{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Onnx: ProtBert-BFD.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyPN7q0EPcg8BgNpeyVojZUf",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a76e635a3b0041ca8a183705b7e30905": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c797edd1845247e6a67e850150dac22e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d7958341c7b64c73b153729451595623",
              "IPY_MODEL_ba170c48aa8147b3a2f94acf5d48bee2"
            ]
          }
        },
        "c797edd1845247e6a67e850150dac22e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d7958341c7b64c73b153729451595623": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4315d50bda1549418de10f1ea82f7228",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 81,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 81,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5e498a1a7b4d481aad0115b0dfe80b24"
          }
        },
        "ba170c48aa8147b3a2f94acf5d48bee2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_efeb8e37a9294dcfb9c58e63a8141ae7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 81.0/81.0 [00:37&lt;00:00, 2.16B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_93d2c77c93014c5eb02bdf946be586bf"
          }
        },
        "4315d50bda1549418de10f1ea82f7228": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5e498a1a7b4d481aad0115b0dfe80b24": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "efeb8e37a9294dcfb9c58e63a8141ae7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "93d2c77c93014c5eb02bdf946be586bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "981bf7d6599a47d99b9b9e2e77bf265a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c655bfab924a4dc08516cdf036d303f0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_57d7a9a8e94046709ba6dacd4401c6fc",
              "IPY_MODEL_0741a1dbcb9644bca56bc51fd2c4be0a"
            ]
          }
        },
        "c655bfab924a4dc08516cdf036d303f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "57d7a9a8e94046709ba6dacd4401c6fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2bf4e056cc404549ad89d5248ec598aa",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cac85d511a884bb59ff245ef06dd529b"
          }
        },
        "0741a1dbcb9644bca56bc51fd2c4be0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0ccbc69421824986a50751a2d45e8056",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:00&lt;00:00, 142B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_89855c033b9d4781b83fa8c9979a5631"
          }
        },
        "2bf4e056cc404549ad89d5248ec598aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cac85d511a884bb59ff245ef06dd529b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0ccbc69421824986a50751a2d45e8056": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "89855c033b9d4781b83fa8c9979a5631": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bb8e95d52192438fafbc3dc0dd1d876e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c09d4f3fa3f04ca1b85e172f5ca548b7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f7dbfb5258a24fb4bf943451e4a7579e",
              "IPY_MODEL_cd57c88e10964a829b605dbf1e17c57c"
            ]
          }
        },
        "c09d4f3fa3f04ca1b85e172f5ca548b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f7dbfb5258a24fb4bf943451e4a7579e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f22e50a0e8e54deca5a40011802e6601",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 86,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 86,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8a519c65758a45d296527e535fa4aee1"
          }
        },
        "cd57c88e10964a829b605dbf1e17c57c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_92552b6be63f419b87fc09586916b359",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 86.0/86.0 [00:35&lt;00:00, 2.41B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b71ea87f03c0465e9ecfd82b6a3130bf"
          }
        },
        "f22e50a0e8e54deca5a40011802e6601": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8a519c65758a45d296527e535fa4aee1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "92552b6be63f419b87fc09586916b359": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b71ea87f03c0465e9ecfd82b6a3130bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f7202449706e4b54a910b80d6b9ef452": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_64cc661ef6ab423e9e1c8f4203f38495",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_25292309e9fc491a8589d3ddf992cee8",
              "IPY_MODEL_a71aa3316a7a4227a4a77691762d71b1"
            ]
          }
        },
        "64cc661ef6ab423e9e1c8f4203f38495": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "25292309e9fc491a8589d3ddf992cee8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_63c47a5bef23432b8020b9ac37d496d6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 361,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 361,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_00f6b0baec924d0088a0d8bd2f16d346"
          }
        },
        "a71aa3316a7a4227a4a77691762d71b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dc3d3416d1bb4679a05b542ad1743955",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 361/361 [00:03&lt;00:00, 117B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a5b1dcce01a64ab0bd79ca4d439a47fb"
          }
        },
        "63c47a5bef23432b8020b9ac37d496d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "00f6b0baec924d0088a0d8bd2f16d346": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dc3d3416d1bb4679a05b542ad1743955": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a5b1dcce01a64ab0bd79ca4d439a47fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da469591010c44dab4f1e4b0d97857cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_487e312f9a7c4091a601386b619dd507",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_efe8e75bec854b108cb368ba9d928d27",
              "IPY_MODEL_a27b329dc8d3429995be00cfe11faabf"
            ]
          }
        },
        "487e312f9a7c4091a601386b619dd507": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "efe8e75bec854b108cb368ba9d928d27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0422643a77744889a424aa4aa118bc52",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1684058277,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1684058277,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_375ecc5d66ea478ba75129234efb233d"
          }
        },
        "a27b329dc8d3429995be00cfe11faabf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_617195279d404a11a024d33529875cbc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.68G/1.68G [00:31&lt;00:00, 53.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aa584c3c63c74c12b9bf7cc56dd538e4"
          }
        },
        "0422643a77744889a424aa4aa118bc52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "375ecc5d66ea478ba75129234efb233d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "617195279d404a11a024d33529875cbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aa584c3c63c74c12b9bf7cc56dd538e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/agemagician/ProtTrans/blob/master/Embedding/Onnx/ProtBert-BFD.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OR9Z36bHYzdA"
      },
      "source": [
        "**Important Notes:**\n",
        "\n",
        "1. The current onnx implementation only works correctly with a single sequence per batch.\n",
        "2. It will work with many sequences per batch, but the results will be different from the original pytorch implementation.\n",
        "3. We will update the notebook if we figured out the problem.\n",
        "4. Pytorch FP16 is much faster than Pytorch FP32 when more sequences per batch is used."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uBLNTSYelrEV"
      },
      "source": [
        "**Check GPU**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9HAW2zl9etR7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5e1511f0-9666-41e5-aa16-4eecd3b37666"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Wed Nov 18 21:52:50 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 455.38       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P0    24W / 300W |      0MiB / 16130MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jyFzGFQ97b_z"
      },
      "source": [
        "**Install libraries**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sF27J8pO4jQZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ceadbb77-4b23-4235-8f75-e5318a88b6be"
      },
      "source": [
        "!pip install -q transformers\n",
        "!pip install -q biopython\n",
        "!pip install -q onnxruntime-gpu==1.4.0\n",
        "!pip install -q onnxruntime-tools==1.4.2"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.3MB 12.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 55.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 28.6MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.9MB 57.4MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 2.3MB 13.9MB/s \n",
            "\u001b[?25hCollecting onnxruntime-gpu==1.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/18/7e/ee1069410b1c955670a1f9889a2d8ce3a2df91fadec0d671b4685ac52519/onnxruntime_gpu-1.4.0-cp36-cp36m-manylinux2010_x86_64.whl (25.4MB)\n",
            "\u001b[K     |████████████████████████████████| 25.4MB 1.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.16.6 in /usr/local/lib/python3.6/dist-packages (from onnxruntime-gpu==1.4.0) (1.18.5)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from onnxruntime-gpu==1.4.0) (3.12.4)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->onnxruntime-gpu==1.4.0) (50.3.2)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf->onnxruntime-gpu==1.4.0) (1.15.0)\n",
            "Installing collected packages: onnxruntime-gpu\n",
            "Successfully installed onnxruntime-gpu-1.4.0\n",
            "Collecting onnxruntime-tools==1.4.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/19/6a/a6159e38c3ad9320f0ecb504ac22b96e7549d1422214cbc73a30ea4e1a5f/onnxruntime_tools-1.4.2-py3-none-any.whl (136kB)\n",
            "\u001b[K     |████████████████████████████████| 143kB 13.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from onnxruntime-tools==1.4.2) (20.4)\n",
            "Collecting py-cpuinfo\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f6/f5/8e6e85ce2e9f6e05040cf0d4e26f43a4718bcc4bce988b433276d4b1a5c1/py-cpuinfo-7.0.0.tar.gz (95kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 9.7MB/s \n",
            "\u001b[?25hCollecting onnx\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2a/82/e8d0fb64df623a3b716145192ed50604f444889778b37e0e9262753d5046/onnx-1.8.0-cp36-cp36m-manylinux2010_x86_64.whl (7.7MB)\n",
            "\u001b[K     |████████████████████████████████| 7.7MB 21.9MB/s \n",
            "\u001b[?25hCollecting coloredlogs\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5c/2f/12747be360d6dea432e7b5dfae3419132cb008535cfe614af73b9ce2643b/coloredlogs-14.0-py2.py3-none-any.whl (43kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 7.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (from onnxruntime-tools==1.4.2) (5.4.8)\n",
            "Collecting py3nvml\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/53/b3/cb30dd8cc1198ae3fdb5a320ca7986d7ca76e23d16415067eafebff8685f/py3nvml-0.2.6-py3-none-any.whl (55kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 9.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from onnxruntime-tools==1.4.2) (1.18.5)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->onnxruntime-tools==1.4.2) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->onnxruntime-tools==1.4.2) (1.15.0)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from onnx->onnxruntime-tools==1.4.2) (3.12.4)\n",
            "Requirement already satisfied: typing-extensions>=3.6.2.1 in /usr/local/lib/python3.6/dist-packages (from onnx->onnxruntime-tools==1.4.2) (3.7.4.3)\n",
            "Collecting humanfriendly>=7.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8e/2d/2f1b0a780b8c948c06c74c8c80e68ac354da52397ba432a1c5ac1923c3af/humanfriendly-8.2-py2.py3-none-any.whl (86kB)\n",
            "\u001b[K     |████████████████████████████████| 92kB 12.5MB/s \n",
            "\u001b[?25hCollecting xmltodict\n",
            "  Downloading https://files.pythonhosted.org/packages/28/fd/30d5c1d3ac29ce229f6bdc40bbc20b28f716e8b363140c26eff19122d8a5/xmltodict-0.12.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->onnx->onnxruntime-tools==1.4.2) (50.3.2)\n",
            "Building wheels for collected packages: py-cpuinfo\n",
            "  Building wheel for py-cpuinfo (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for py-cpuinfo: filename=py_cpuinfo-7.0.0-cp36-none-any.whl size=20071 sha256=7abab0f8ff8098a4b378fc611c783f3e8589a1916cf3f81ea929d10d0fd48461\n",
            "  Stored in directory: /root/.cache/pip/wheels/f1/93/7b/127daf0c3a5a49feb2fecd468d508067c733fba5192f726ad1\n",
            "Successfully built py-cpuinfo\n",
            "Installing collected packages: py-cpuinfo, onnx, humanfriendly, coloredlogs, xmltodict, py3nvml, onnxruntime-tools\n",
            "Successfully installed coloredlogs-14.0 humanfriendly-8.2 onnx-1.8.0 onnxruntime-tools-1.4.2 py-cpuinfo-7.0.0 py3nvml-0.2.6 xmltodict-0.12.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gChn9igf7gS4"
      },
      "source": [
        "**Import libraries**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iQHK1bQb6e1c"
      },
      "source": [
        "import torch\n",
        "from transformers import BertModel, BertTokenizer, pipeline\n",
        "from pathlib import Path\n",
        "from transformers.convert_graph_to_onnx import convert\n",
        "\n",
        "import numpy as np\n",
        "import os\n",
        "from tqdm.auto import tqdm\n",
        "import math\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import re\n",
        "\n",
        "from contextlib import contextmanager\n",
        "from dataclasses import dataclass\n",
        "from time import time\n",
        "from tqdm import trange\n",
        "\n",
        "from os import environ\n",
        "from psutil import cpu_count\n",
        "\n",
        "# optimize transformer-based models with onnxruntime-tools\n",
        "from onnxruntime_tools import optimizer\n",
        "# This is optional if you need more optimization for bert based models\n",
        "from onnxruntime_tools.transformers.onnx_model_bert import BertOptimizationOptions\n",
        "from onnxruntime import GraphOptimizationLevel, InferenceSession, SessionOptions, get_all_providers"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nMJA0BvZd1RP"
      },
      "source": [
        "%matplotlib inline"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4YobGkV0l6LK"
      },
      "source": [
        "**Create Some Data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ajyc9G9K20x"
      },
      "source": [
        "proteins = [\"VKAPHEHEHEDNPSVSRVFNRLSQARQNARSIIWGGDGELLTGLIRMVLQVTSPDDKKLSQRDEDGPRLNNCIFTIGFLVPYISEKLKALVTLEVGKPAWTAGDRRKVMGHCADEQIRLILTCAFVGYHTAYKEKFTSEAGSELMGLAFDKYDESYTVEMKKITPFGSSRLVIVLLQFNMQADNAPANAPLAVQSIYGIT\",\n",
        "                 \"TYGASGLERPGYIIDSAQLSLRYGADADRAARFPNISAMRVGTLCTLGSLPKQTTNPINFQKIVDSGAPKDDGDGWERKWLTGCIANPYTILGTPTSGEGKDQRPQDGSRVNISEHLQTV PAAVQSQGESWEVGEEMKLKRESIIRVQPELTLSILGDKQARTLPAHSMKKSLEPAARVIMNLLPRRHEKVFCEPTDRRAQELIELAMERNNYDMLELSPHAPDTPRLAVEEAVAAINARLAFVGLGDRGNESGLFYVVDKGDKFDDEYSSYINIVGIQKLWKGSEVLKWVSGVGSMTHNEIRPGTCCNEPQDMFIQELVHLTTLFNGTVRGGVKDFKSLTFSALLEAEAVSAKSIFTRLRLHMKIYLYNISHDHFSADVLDQYKLGFAAEYKHRAALPENKKLTLDLYSAFFSTEPQSIGANIVMEASKGQVPITTMLIINKRISVLGAALALQVELKNFYNNRLRVFDQGEYPCAKLKVLWFTMEHHGCDIFVKVFVTAPDVQDEVAIVS\",\n",
        "                 \"PSKSLKCTGGNAFALRFIRQDDAEVAPLGVSEIWLNDIGMKHELYQTVRT\",\n",
        "                 \"IKNEIVHTVFSTLQPRDHLNGLRLQIEEATCDALRVPLGAGGLVAEEVSKKRPALYDANYFEDVLKTAGIFSPRVSGRADMEKQGFNPKMTSLQSYWSII\"]"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6gjzYgQYf5Fi"
      },
      "source": [
        "proteins = [\" \".join(list(item)) for item in proteins]"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bpFpdM3Af_aN"
      },
      "source": [
        "proteins = [re.sub(r\"[UZOB]\", \"X\", sequence) for sequence in proteins]"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WtCkXVMSmPf4"
      },
      "source": [
        "**Import Tokenizer**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3LsHfgaOmNGF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164,
          "referenced_widgets": [
            "a76e635a3b0041ca8a183705b7e30905",
            "c797edd1845247e6a67e850150dac22e",
            "d7958341c7b64c73b153729451595623",
            "ba170c48aa8147b3a2f94acf5d48bee2",
            "4315d50bda1549418de10f1ea82f7228",
            "5e498a1a7b4d481aad0115b0dfe80b24",
            "efeb8e37a9294dcfb9c58e63a8141ae7",
            "93d2c77c93014c5eb02bdf946be586bf",
            "981bf7d6599a47d99b9b9e2e77bf265a",
            "c655bfab924a4dc08516cdf036d303f0",
            "57d7a9a8e94046709ba6dacd4401c6fc",
            "0741a1dbcb9644bca56bc51fd2c4be0a",
            "2bf4e056cc404549ad89d5248ec598aa",
            "cac85d511a884bb59ff245ef06dd529b",
            "0ccbc69421824986a50751a2d45e8056",
            "89855c033b9d4781b83fa8c9979a5631",
            "bb8e95d52192438fafbc3dc0dd1d876e",
            "c09d4f3fa3f04ca1b85e172f5ca548b7",
            "f7dbfb5258a24fb4bf943451e4a7579e",
            "cd57c88e10964a829b605dbf1e17c57c",
            "f22e50a0e8e54deca5a40011802e6601",
            "8a519c65758a45d296527e535fa4aee1",
            "92552b6be63f419b87fc09586916b359",
            "b71ea87f03c0465e9ecfd82b6a3130bf"
          ]
        },
        "outputId": "bb1a4aab-1f77-41ab-f3c1-deaf16596dfe"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained(\"Rostlab/prot_bert_bfd\", do_lower_case=False)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a76e635a3b0041ca8a183705b7e30905",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=81.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "981bf7d6599a47d99b9b9e2e77bf265a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bb8e95d52192438fafbc3dc0dd1d876e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=86.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LjT2fDAH7jFD"
      },
      "source": [
        "**Convert pytorch to onnx**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQf6BBWn6eu9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 407,
          "referenced_widgets": [
            "f7202449706e4b54a910b80d6b9ef452",
            "64cc661ef6ab423e9e1c8f4203f38495",
            "25292309e9fc491a8589d3ddf992cee8",
            "a71aa3316a7a4227a4a77691762d71b1",
            "63c47a5bef23432b8020b9ac37d496d6",
            "00f6b0baec924d0088a0d8bd2f16d346",
            "dc3d3416d1bb4679a05b542ad1743955",
            "a5b1dcce01a64ab0bd79ca4d439a47fb",
            "da469591010c44dab4f1e4b0d97857cb",
            "487e312f9a7c4091a601386b619dd507",
            "efe8e75bec854b108cb368ba9d928d27",
            "a27b329dc8d3429995be00cfe11faabf",
            "0422643a77744889a424aa4aa118bc52",
            "375ecc5d66ea478ba75129234efb233d",
            "617195279d404a11a024d33529875cbc",
            "aa584c3c63c74c12b9bf7cc56dd538e4"
          ]
        },
        "outputId": "75437b93-e1ab-4816-899b-87e84f62c0fb"
      },
      "source": [
        "!rm -rf onnx/\n",
        "# Handles all the above steps for you\n",
        "convert(framework=\"pt\",\n",
        "        model=\"Rostlab/prot_bert_bfd\",\n",
        "        output=Path(\"onnx/prot_bert_bfd.onnx\"),\n",
        "        opset=12,\n",
        "        tokenizer=\"Rostlab/prot_bert_bfd\")"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ONNX opset version set to: 12\n",
            "Loading pipeline (model: Rostlab/prot_bert_bfd, tokenizer: Rostlab/prot_bert_bfd)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f7202449706e4b54a910b80d6b9ef452",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=361.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "da469591010c44dab4f1e4b0d97857cb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1684058277.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Creating folder onnx\n",
            "Using framework PyTorch: 1.7.0+cu101\n",
            "Found input input_ids with shape: {0: 'batch', 1: 'sequence'}\n",
            "Found input token_type_ids with shape: {0: 'batch', 1: 'sequence'}\n",
            "Found input attention_mask with shape: {0: 'batch', 1: 'sequence'}\n",
            "Found output output_0 with shape: {0: 'batch', 1: 'sequence'}\n",
            "Found output output_1 with shape: {0: 'batch'}\n",
            "Ensuring inputs are in correct order\n",
            "position_ids is not present in the generated input list.\n",
            "Generated inputs order: ['input_ids', 'attention_mask', 'token_type_ids']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/modeling_bert.py:191: TracerWarning: Converting a tensor to a Python index might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
            "  position_ids = self.position_ids[:, :seq_length]\n",
            "/usr/local/lib/python3.6/dist-packages/transformers/modeling_utils.py:1673: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
            "  input_tensor.shape == tensor_shape for input_tensor in input_tensors\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K089DVtOMoug"
      },
      "source": [
        "**Optimize onnx model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OdAaAhZoDJ6Y"
      },
      "source": [
        "# disable embedding layer norm optimization for better model size reduction\n",
        "opt_options = BertOptimizationOptions('bert')\n",
        "opt_options.enable_embed_layer_norm = False\n",
        "\n",
        "opt_model = optimizer.optimize_model(\n",
        "    'onnx/prot_bert_bfd.onnx',\n",
        "    'bert', \n",
        "    num_heads=16,\n",
        "    hidden_size=1024,\n",
        "    optimization_options=opt_options)\n",
        "# This is optional if you need to run the model using mixed precisionn\n",
        "opt_model.convert_model_float32_to_float16()\n",
        "\n",
        "opt_model.save_model_to_file('onnx/prot_bert_bfd.opt.onnx')"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KBYYKuBs7rk5"
      },
      "source": [
        "**Embed with onnx**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xbf-9xAvcmmq"
      },
      "source": [
        "inference_time = {\"Pytorch FP32\":0,\"Pytorch FP16\":0, \"Onnx FP16\":0}"
      ],
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2dWnJOjO6l9V"
      },
      "source": [
        "# Constants from the performance optimization available in onnxruntime\n",
        "# It needs to be done before importing onnxruntime\n",
        "environ[\"OMP_NUM_THREADS\"] = str(cpu_count(logical=True))\n",
        "environ[\"OMP_WAIT_POLICY\"] = 'ACTIVE'"
      ],
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jn5_vunt7vfG"
      },
      "source": [
        "def create_model_for_provider(model_path: str, provider: str) -> InferenceSession: \n",
        "  \n",
        "  assert provider in get_all_providers(), f\"provider {provider} not found, {get_all_providers()}\"\n",
        "\n",
        "  # Few properties that might have an impact on performances (provided by MS)\n",
        "  options = SessionOptions()\n",
        "  options.intra_op_num_threads = 1\n",
        "  options.graph_optimization_level = GraphOptimizationLevel.ORT_ENABLE_ALL\n",
        "\n",
        "  # Load the model as a graph and prepare the CPU backend \n",
        "  session = InferenceSession(model_path, options, providers=[provider])\n",
        "  session.disable_fallback()\n",
        "    \n",
        "  return session\n",
        "\n",
        "\n",
        "@contextmanager\n",
        "def track_infer_time(buffer: [int]):\n",
        "    start = time()\n",
        "    yield\n",
        "    end = time()\n",
        "\n",
        "    buffer.append(end - start)\n",
        "\n",
        "\n",
        "@dataclass\n",
        "class OnnxInferenceResult:\n",
        "  model_inference_time: [int]  \n",
        "  optimized_model_path: str"
      ],
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ndmmlVw7ysv"
      },
      "source": [
        "gpu_model = create_model_for_provider('/content/onnx/prot_bert_bfd.opt.onnx', \"CUDAExecutionProvider\")"
      ],
      "execution_count": 70,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3oIvSz-g9KUK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0d199377-e7ac-455c-cd24-dc23355c8a2b"
      },
      "source": [
        "totalDuration = 0\n",
        "\n",
        "for itr in range(50):\n",
        "  for batch in proteins:\n",
        "    maxLen = len(max(batch, key=len))/2\n",
        "    minLen = len(min(batch, key=len))/2\n",
        "    batch_size = len(batch)\n",
        "\n",
        "    time1 = time()\n",
        "\n",
        "    # Inputs are provided through numpy array\n",
        "    model_inputs = tokenizer(batch, return_tensors=\"pt\", padding=True, add_special_tokens=True)\n",
        "    inputs_onnx = {k: v.cpu().detach().numpy() for k, v in model_inputs.items()}\n",
        "\n",
        "    # Run the model (None = get all the outputs)\n",
        "    onnx_embedding, onnx_pooled = gpu_model.run(None, inputs_onnx)\n",
        "\n",
        "    #embedding = model(batch)\n",
        "    time2 = time()\n",
        "\n",
        "    duration = time2-time1\n",
        "    totalDuration += duration\n",
        "    print('{:s} model took {:.3f} ms for sequence length between {:.3f} and {:3f} of total sequences {:d}'.format('ProtBert-BFD', duration*1000.0,minLen,maxLen,batch_size))\n",
        "\n",
        "inference_time[\"Onnx FP16\"] = totalDuration*1000.0\n",
        "print('Total duration is {:.3f}'.format(totalDuration*1000.0))"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ProtBert-BFD model took 13.136 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 25.217 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 6.183 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 7.200 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.950 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 22.419 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.861 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.861 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 10.051 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 22.913 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.715 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.793 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 10.017 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 22.711 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.833 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 7.113 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 10.039 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 22.182 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.637 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.737 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.511 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.313 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.588 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.813 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.628 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.595 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.395 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.649 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.443 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.182 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.544 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.656 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.417 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.235 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.493 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.469 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.598 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.683 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.805 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.751 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.487 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.969 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.369 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.715 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.501 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.855 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.544 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.699 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.399 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.520 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.749 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.814 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.554 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.513 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.655 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.648 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.515 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.929 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.702 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.619 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.557 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.555 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.354 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.706 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.514 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.775 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.613 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.704 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.563 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.884 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.577 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.842 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.484 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.718 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.602 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.611 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.551 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.981 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.757 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.769 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.754 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.454 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.633 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.717 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.591 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.501 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.636 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.796 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.148 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.086 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.519 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.669 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.422 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 22.726 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.709 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.562 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.630 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.638 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.629 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.665 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.286 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.055 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.484 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.581 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.399 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.834 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.653 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.785 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.213 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.182 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.313 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.423 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.461 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.513 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.559 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.813 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.487 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.418 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.657 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.739 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.477 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.966 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.489 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.606 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.527 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.364 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.540 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.713 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.207 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.362 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.417 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.447 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.240 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 22.312 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.610 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.774 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.322 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.981 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.669 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.648 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.510 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.569 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.600 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.757 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.274 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.572 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.621 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.770 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.578 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.694 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.604 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.667 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.468 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.439 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.594 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.725 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.321 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.832 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.735 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.724 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.409 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.482 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.384 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.829 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.429 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.533 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.515 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.780 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.514 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.719 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.690 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.785 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.562 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.741 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.696 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.791 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.596 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.854 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.606 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.628 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.251 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.286 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.305 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.765 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.104 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.352 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.550 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.527 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.380 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.528 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.532 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.674 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.409 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.523 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.574 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.618 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 9.328 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 21.488 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 5.599 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 6.625 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "Total duration is 2182.240\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c2tLZNiw9lbJ"
      },
      "source": [
        "**Embed with pytorch**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0gXDHwew9Z4G"
      },
      "source": [
        "model = BertModel.from_pretrained(\"Rostlab/prot_bert_bfd\")"
      ],
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TFIVBbYk904w"
      },
      "source": [
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EeShhlYU936k"
      },
      "source": [
        "model = model.to(device)\n",
        "model = model.eval()"
      ],
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EG1A5ivC-NHG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cb407193-9901-4564-a113-266fb323ccd8"
      },
      "source": [
        "totalDuration = 0\n",
        "\n",
        "for itr in range(50):\n",
        "  for batch in proteins:\n",
        "    #batch = list_part[0]\n",
        "    maxLen = len(max(batch, key=len))/2\n",
        "    minLen = len(min(batch, key=len))/2\n",
        "    batch_size = len(batch)\n",
        "\n",
        "    time1 = time()\n",
        "\n",
        "    #ids = tokenizer.encode_plus(batch, return_tensors=\"pt\", add_special_tokens=False, pad_to_max_length=True)\n",
        "    ids = tokenizer(batch, return_tensors=\"pt\", padding=True, add_special_tokens=True)\n",
        "    input_ids = torch.tensor(ids['input_ids']).to(device)\n",
        "    with torch.no_grad():\n",
        "      pytorch_embedding, pytorch_pool = model(input_ids=input_ids)\n",
        "\n",
        "    #embedding = model(batch)\n",
        "    time2 = time()\n",
        "\n",
        "    duration = time2-time1\n",
        "    totalDuration += duration\n",
        "    print('{:s} model took {:.3f} ms for sequence length between {:.3f} and {:3f} of total sequences {:d}'.format('ProtBert-BFD', duration*1000.0,minLen,maxLen,batch_size))\n",
        "\n",
        "inference_time[\"Pytorch FP32\"] = totalDuration*1000.0\n",
        "print('Total duration is {:.3f}'.format(totalDuration*1000.0))"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:14: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "ProtBert-BFD model took 35.942 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 39.580 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 50.407 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 35.005 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 36.307 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.634 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 47.833 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.729 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.626 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.522 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.473 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.457 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.494 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.049 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.234 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.168 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.055 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.576 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.720 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.244 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.461 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.996 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.993 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.359 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.225 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.627 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 46.981 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.201 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.853 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.910 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.795 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.341 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.560 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.320 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.465 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 35.237 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 37.552 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.963 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.039 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.394 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.086 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 32.926 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 50.653 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.667 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.587 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 37.762 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 43.607 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.792 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 34.463 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.667 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.203 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.049 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.138 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.726 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 52.834 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.268 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.738 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.874 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 47.735 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.077 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 34.381 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.683 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.035 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.890 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.436 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.684 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 55.855 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.347 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.914 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.564 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.008 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.481 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 35.641 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.151 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.709 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.444 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.901 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.054 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 50.744 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.373 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.449 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.685 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 46.843 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.303 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.531 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.090 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 47.968 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.561 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.736 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.002 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 50.269 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.790 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.250 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.341 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 46.078 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.517 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.660 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.716 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.140 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.187 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.389 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.763 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 50.606 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.259 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.222 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.643 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 49.334 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.490 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.912 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.224 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 46.360 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.868 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.091 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.786 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 51.809 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.451 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.357 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.512 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 47.993 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.561 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.182 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.550 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.551 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.401 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 36.287 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.350 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 50.995 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 34.366 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 37.846 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.423 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 47.916 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.825 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.327 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.767 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 47.549 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.085 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.597 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.964 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 49.800 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.766 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 30.647 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.075 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 47.917 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.750 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.669 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.827 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 51.304 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.444 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.216 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.139 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.250 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.862 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.102 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.110 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 52.060 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.538 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 40.014 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 40.806 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 42.159 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 34.527 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 36.335 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.470 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 47.101 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.237 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.737 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.134 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 50.230 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.730 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 34.933 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 32.945 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 50.659 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.781 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.168 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.233 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 51.054 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.895 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.211 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.146 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 51.286 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.867 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.356 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.250 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 46.429 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.478 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.341 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.490 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 49.356 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.863 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.687 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.814 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 46.618 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.501 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 35.631 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.314 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 48.704 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.335 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.770 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 40.149 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 47.313 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 29.752 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "Total duration is 7395.486\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f0jhwvpFUp6v"
      },
      "source": [
        "model_half = model.half()"
      ],
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tnGFufpyVTVK",
        "outputId": "67101f8e-55ab-448d-eb3b-f90872fa8360"
      },
      "source": [
        "totalDuration = 0\n",
        "\n",
        "for itr in range(50):\n",
        "  for batch in proteins:\n",
        "    #batch = list_part[0]\n",
        "    maxLen = len(max(batch, key=len))/2\n",
        "    minLen = len(min(batch, key=len))/2\n",
        "    batch_size = len(batch)\n",
        "\n",
        "    time1 = time()\n",
        "\n",
        "    #ids = tokenizer.encode_plus(batch, return_tensors=\"pt\", add_special_tokens=False, pad_to_max_length=True)\n",
        "    ids = tokenizer(batch, return_tensors=\"pt\", padding=True, add_special_tokens=True)\n",
        "    input_ids = torch.tensor(ids['input_ids']).to(device)\n",
        "    with torch.no_grad():\n",
        "      pytorch_embedding_half, pytorch_pool_half = model_half(input_ids=input_ids)\n",
        "\n",
        "    #embedding = model(batch)\n",
        "    time2 = time()\n",
        "\n",
        "    duration = time2-time1\n",
        "    totalDuration += duration\n",
        "    print('{:s} model took {:.3f} ms for sequence length between {:.3f} and {:3f} of total sequences {:d}'.format('ProtBert-BFD', duration*1000.0,minLen,maxLen,batch_size))\n",
        "\n",
        "inference_time[\"Pytorch FP16\"] = totalDuration*1000.0\n",
        "print('Total duration is {:.3f}'.format(totalDuration*1000.0))"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ProtBert-BFD model took 36.103 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.940 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 31.531 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:14: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "ProtBert-BFD model took 33.623 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.002 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.502 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.243 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 35.491 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.760 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.886 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.575 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.060 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.951 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.726 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 35.975 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.471 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 36.586 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.862 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 31.850 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.098 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 35.177 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.519 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 33.119 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 36.454 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.867 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.516 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 36.528 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.507 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.323 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.549 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.659 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.049 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.324 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 38.379 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 35.182 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 34.157 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.310 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 37.001 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 31.869 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.424 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.524 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.470 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.060 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.401 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 34.453 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 37.726 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 36.006 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 37.669 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.982 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.182 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 30.783 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.913 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.702 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.745 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 33.897 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.584 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.517 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.111 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.327 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.681 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.854 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.059 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.011 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.046 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.309 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 38.068 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 34.371 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.203 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.984 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.620 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 30.543 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.314 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 35.154 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.671 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 33.013 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.218 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.149 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.228 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.105 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 34.622 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.227 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.963 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 33.291 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.312 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.257 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.114 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 35.922 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 39.515 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 35.597 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.842 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 31.142 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.631 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.374 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.998 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 34.755 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.213 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 39.404 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.310 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 33.593 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.430 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 34.378 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.633 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.691 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.269 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.829 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 37.849 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 34.629 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 31.940 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.223 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.350 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 31.792 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.807 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.164 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 37.683 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 30.142 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.083 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.478 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.116 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 30.546 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 34.031 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.515 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.013 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 30.988 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 34.502 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 35.138 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.342 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.260 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.789 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.103 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 32.859 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 31.458 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.212 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 35.822 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.823 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.372 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.506 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 34.108 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.740 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 31.291 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.605 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.987 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 37.372 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.355 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.293 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.705 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.303 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 33.960 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.481 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 34.476 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.578 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.485 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.623 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.355 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 37.265 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 33.106 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 35.181 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 37.613 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.763 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.497 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.633 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.854 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 36.352 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.380 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.439 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 37.305 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 40.353 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 32.064 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.154 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.442 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.875 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 31.099 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.180 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 37.010 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.560 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 30.011 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 30.719 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 33.833 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.298 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 30.834 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.158 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.446 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 33.086 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 31.758 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 33.870 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 32.066 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 35.054 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 35.363 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.351 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 31.697 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 34.277 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 35.598 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 34.105 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 34.938 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 39.792 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 36.062 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 35.719 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "ProtBert-BFD model took 34.979 ms for sequence length between 0.500 and 0.500000 of total sequences 399\n",
            "ProtBert-BFD model took 39.084 ms for sequence length between 0.500 and 0.500000 of total sequences 1025\n",
            "ProtBert-BFD model took 35.279 ms for sequence length between 0.500 and 0.500000 of total sequences 99\n",
            "ProtBert-BFD model took 32.861 ms for sequence length between 0.500 and 0.500000 of total sequences 199\n",
            "Total duration is 6771.729\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1_FEz3t4gxxF"
      },
      "source": [
        "**Compare Inference Performance**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "epWQDcAKdTZf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 777
        },
        "outputId": "2269a25e-4454-4aa8-ef34-8c22713e5fa5"
      },
      "source": [
        "plt.rcdefaults()\n",
        "fig, ax = plt.subplots(figsize=(16, 12))\n",
        "ax.set_ylabel(\"Total Inference time (ms)\")\n",
        "ax.set_title(\"Total inference time (ms) for each library\")\n",
        "ax.bar(inference_time.keys(), inference_time.values())#, yerr=time_results_std)\n",
        "plt.show()"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABSwAAAPeCAYAAAAGYVTMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZxVdeH/8fcMOCMoMxixiBKCG0uWSYnkgluQomZuD9QUBTWV6gtqLmWkuOCSktrXPUENy6UylxShcMclUlNRQwX1p4GKwriwKHN/f/SY+3UcNnGZUz6fj8d9PLjnfO45n3O5M9irc8+pKJVKpQAAAAAAFEBlc08AAAAAAKCBYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAPxHufPOO1NRUZE777xzuePGjx+fioqKzJo1a5X2M2PGjAwYMCC1tbWpqKjIjTfeuErb+W+37bbbZtttt23uaTRx1llnpUePHqmvr//M9jl37tysscYa+fOf/7zSr/k8fM623XbbfPnLX17l16+33no56KCDys+X9jvg4+4DACgWwRIAWKGKioqVeqwoIibJ6aef/h8RZYYMGZLHH388p512Wq6++up8/etfb+4pNZvp06fnpJNOWuX4+1mrq6vLmWeemeOOOy6VlZ/df+62a9cuhxxySH72s5+t9Gt8zgAAmmrZ3BMAAIrv6quvbvT8qquuyqRJk5os79mz5wq3dfrpp2evvfbK7rvv/onO8cMOOOCADB48ONXV1R/5tQsWLMjUqVPz05/+ND/4wQ8+hdn9Z5k+fXpOPvnkbLvttllvvfUarbvjjjuaZ1LLccUVV+T999/Pvvvu+5nv+/DDD8/555+fv/71r9l+++2XO9bnbNVss802WbBgQaqqqpp7KgDAp0SwBABW6Hvf+16j5w888EAmTZrUZHmRtGjRIi1atFil17722mtJkrZt235i81m4cGGqqqo+0zP+PgtFjEbjxo3LbrvtltVXX/0z33fPnj3z5S9/OePHj19hsPw0PmfvvPNO1lhjjU9se0VUWVn5if7dfhI/m/+tP98A0Fz8iwoAfCLeeeedHH300enSpUuqq6uz8cYb5xe/+EVKpVJ5TEVFRd55551ceeWV5a+RN1yb7oUXXsiRRx6ZjTfeOK1atUq7du2y9957r/LXkJd2Dcv11lsvu+yyS+69995svvnmWX311dO9e/dcddVV5TEnnXRSunbtmiT58Y9/nIqKikZnFb788ssZOnRoOnbsmOrq6vTu3TtXXHFFo303XGPvd7/7XU488cSss846ad26derq6pIkDz74YL797W+ntrY2rVu3Tv/+/XPfffc12sZJJ52UioqKPPvssznooIPStm3b1NbW5uCDD867777b5Hh/85vfZPPNN0/r1q2z1lprZZtttmly9uNtt92WrbfeOmussUbatGmTQYMG5cknn1zh+7j33nsnSbbbbrsmX///8DUsG479uuuuy8knn5x11lknbdq0yV577ZX58+dn0aJFGTFiRDp06JA111wzBx98cBYtWrTU4+nTp09atWqVL3zhCxk8eHBeeuml5c41SWbOnJl//OMf2XHHHRstnzVrVioqKvKLX/wi//u//5vu3bundevWGTBgQF566aWUSqWccsopWXfdddOqVat85zvfyRtvvNFoG3/7298ycODAfPGLX0yrVq3SrVu3DB06tMkcvvWtb+Xmm29u9Nn/sBV9zh555JHstNNOqampyZprrpkddtghDzzwQKNtNHzG77rrrhx55JHp0KFD1l133eW+P4sWLcrPf/7zbLDBBqmurk6XLl1y7LHHNvk7GDduXLbffvt06NAh1dXV6dWrVy666KKlbvO2225L//7906ZNm9TU1OQb3/hGrrnmmibjpk+fnu222y6tW7fOOuusk7POOmu5c12W5V3Hdtq0afnmN79Z/vu5+OKLl/rapf1svvHGGznmmGOyySabZM0110xNTU122mmnPPbYYyu1jUcffTQVFRUZO3Zsk3ndf//9qaioyG9/+9tVOmYA+LxxhiUA8LGVSqXstttumTJlSoYNG5ZNN900EydOzI9//OO8/PLL5f8Bf/XVV+eQQw7J5ptvnsMOOyxJsv766ydJHn744dx///0ZPHhw1l133cyaNSsXXXRRtt1220yfPj2tW7f+ROb67LPPZq+99sqwYcMyZMiQXHHFFTnooIPSp0+f9O7dO3vssUfatm2bkSNHZt99983OO++cNddcM0kyZ86cbLHFFqmoqMgPfvCDtG/fPrfddluGDRuWurq6jBgxotG+TjnllFRVVeWYY47JokWLUlVVlb/+9a/Zaaed0qdPn/z85z9PZWVlOQ7dc8892XzzzRttY5999km3bt0yZsyY/P3vf8/ll1+eDh065MwzzyyPOfnkk3PSSSflm9/8ZkaPHp2qqqo8+OCD+etf/5oBAwaU3/shQ4Zk4MCBOfPMM/Puu+/moosuylZbbZVHHnmkyVe9G2yzzTb50Y9+lPPPPz8/+clPyl/7X9HX/8eMGZNWrVrl+OOPz7PPPpsLLrggq622WiorK/Pmm2/mpJNOygMPPJDx48enW7duGTVqVPm1p512Wn72s59ln332ySGHHJLXXnstF1xwQbbZZps88sgjyz0j8f7770+SbLbZZktdP2HChCxevDg//OEP88Ybb+Sss87KPvvsk+233z533nlnjjvuuPJ8jznmmHKMfvXVVzNgwIC0b98+xx9/fNq2bZtZs2blD3/4Q5N99OnTJ2PHjs2TTz65zBvBLO9z9uSTT2brrbdOTU1Njj322Ky22mq55JJLsu222+auu+5K3759G23ryCOPTPv27TNq1Ki88847y3xv6uvrs9tuu+Xee+/NYYcdlp49e+bxxx/P2LFj889//rPRtWUvuuii9O7dO7vttltatmyZm2++OUceeWTq6+szfPjw8rjx48dn6NCh6d27d0444YS0bds2jzzySG6//fbst99+5XFvvvlmvv3tb2ePPfbIPvvskxtuuCHHHXdcNtlkk+y0007LnPNH8eabb2bnnXfOPvvsk3333TfXXXddjjjiiFRVVTUJy0v72Zw+fXpuvPHG7L333unWrVvmzJmTSy65JP3798/06dPTuXPn5W6jR48e2XLLLTNhwoSMHDmy0dgJEyakTZs2+c53vvOJHCsA/NcrAQB8RMOHDy998D8jbrzxxlKS0qmnntpo3F577VWqqKgoPfvss+Vla6yxRmnIkCFNtvnuu+82WTZ16tRSktJVV11VXjZlypRSktKUKVOWO8dx48aVkpRmzpxZXta1a9dSktLdd99dXvbqq6+WqqurS0cffXR52cyZM0tJSmeffXajbQ4bNqy09tprl15//fVGywcPHlyqra0tH0PDHLt3797ouOrr60sbbrhhaeDAgaX6+vpGx96tW7fSt771rfKyn//856UkpaFDhzba13e/+91Su3btys9nzJhRqqysLH33u98tLVmypNHYhn289dZbpbZt25YOPfTQRutnz55dqq2tbbL8w66//vplvuf9+/cv9e/fv/y84di//OUvlxYvXlxevu+++5YqKipKO+20U6PX9+vXr9S1a9fy81mzZpVatGhROu200xqNe/zxx0stW7ZssvzDTjzxxFKS0ltvvdVoecPfafv27Uvz5s0rLz/hhBNKSUpf/epXS++9916j+VZVVZUWLlxYKpVKpT/+8Y+lJKWHH354ufsvlUql+++/v5SkdO211y533LI+Z7vvvnupqqqq9Nxzz5WXvfLKK6U2bdqUttlmm/Kyhs/4VlttVXr//fdXOK+rr766VFlZWbrnnnsaLb/44otLSUr33XdfednSfh4HDhxY6t69e/n5vHnzSm3atCn17du3tGDBgkZjP/j57t+/f5Of40WLFpU6depU2nPPPVc4765duzb6nbG03wEN+zjnnHMa7WPTTTctdejQofxZXNbPZqlUKi1cuLDJz9DMmTNL1dXVpdGjRzfZ/9K2cckll5SSlJ566qnyssWLF5e++MUvLvX3HgCwdL4SDgB8bH/+85/TokWL/OhHP2q0/Oijj06pVMptt922wm20atWq/Of33nsvc+fOzQYbbJC2bdvm73//+yc21169emXrrbcuP2/fvn023njjPP/888t9XalUyu9///vsuuuuKZVKef3118uPgQMHZv78+U3mOWTIkEbH9eijj2bGjBnZb7/9Mnfu3PLr33nnneywww65++67U19f32gbhx9+eKPnW2+9debOnVv+evmNN96Y+vr6jBo1qsn18yoqKpIkkyZNyrx587Lvvvs2mneLFi3St2/fTJkyZSXfvZV34IEHZrXVVis/79u3b0qlUpMz3fr27ZuXXnop77//fpLkD3/4Q+rr67PPPvs0mmunTp2y4YYbrnCuc+fOTcuWLctnK37Y3nvvndra2kb7T/59ndaWLVs2Wr548eK8/PLLSf7vOpO33HJL3nvvveXOYa211kqSvP7668sdtzRLlizJHXfckd133z3du3cvL1977bWz33775d577y3/3Tc49NBDV+p6rddff3169uyZHj16NHpvG661+cH39oOf2/nz5+f1119P//798/zzz2f+/PlJ/v25euutt3L88cc3uaZkw2evwZprrtnomrdVVVXZfPPNV/hz91G0bNky3//+9xvt4/vf/35effXVTJs2rdHYD/9sJkl1dXX5Z2jJkiWZO3du1lxzzWy88cZL/R20tG3ss88+WX311TNhwoTysokTJ+b1118v9DV/AaBofCUcAPjYXnjhhXTu3Dlt2rRptLzha8MvvPDCCrexYMGCjBkzJuPGjcvLL7/c6Pp/DYHkk/ClL32pybK11lorb7755nJf99prr2XevHm59NJLc+mlly51zKuvvtroebdu3Ro9nzFjRpJ/h45lmT9/fjl4LW2+DevefPPN1NTU5LnnnktlZWV69eq1zG027HdZN4GpqalZ5mtX1Yfn3RAJu3Tp0mR5fX195s+fn3bt2mXGjBkplUrZcMMNl7rdD0bQT3teScqfi/79+2fPPffMySefnLFjx2bbbbfN7rvvnv3226/JnegbPrsfjnYr47XXXsu7776bjTfeuMm6nj17pr6+Pi+99FJ69+5dXv7hz9myzJgxI0899VTat2+/1PUf/Pzed999+fnPf56pU6c2uWbq/PnzU1tbm+eeey5Jlvm19w9ad911m7wfa621Vv7xj3+s1NxXRufOnZvccGijjTZK8u9rmG6xxRbl5Ut7z+rr63PeeeflwgsvzMyZM7NkyZLyunbt2jUZv7RttG3bNrvuumuuueaanHLKKUn+/XXwddZZZ4U3YQIA/o9gCQAUwg9/+MOMGzcuI0aMSL9+/VJbW5uKiooMHjy4yVmHH8eyzkQrLecGKUnKc/je9763zOD4la98pdHzD5991bCNs88+O5tuuulSt/HhMwNXdb5L2+/VV1+dTp06NVn/wTMLPynLmveKjqe+vj4VFRW57bbbljp2WWdONmjXrl3ef//9vPXWW00C+seZV0VFRW644YY88MADufnmmzNx4sQMHTo055xzTh544IFG82qInF/84heXO9dPyoc/Z8tSX1+fTTbZJOeee+5S1zdE2+eeey477LBDevTokXPPPTddunRJVVVV/vznP2fs2LGr9PP4SXyOP0lLe89OP/30/OxnP8vQoUNzyimn5Atf+EIqKyszYsSIpR7zst73Aw88MNdff33uv//+bLLJJrnpppty5JFHuoM4AHwEgiUA8LF17do1kydPbhKJnn766fL6Bss66+yGG27IkCFDcs4555SXLVy4MPPmzfuUZv3RtG/fPm3atMmSJUua3IF6ZTXcYKimpmaVt7G0bdbX12f69OnLjKAN++3QocMq7XdVzhRcVeuvv35KpVK6detWPjvuo+jRo0eSf98t/MMB+ZOwxRZbZIsttshpp52Wa665Jvvvv39+97vf5ZBDDimPmTlzZpIV35hoadq3b5/WrVvnmWeeabLu6aefTmVlZZOzQVfW+uuvn8ceeyw77LDDcv9Ob7755ixatCg33XRTozNSP/x1/IbP1RNPPJENNthgleb0SXrllVfyzjvvNDrL8p///GeSLPOmUh90ww03ZLvttsuvf/3rRsvnzZv3keLzt7/97bRv3z4TJkxI37598+677+aAAw5Y6dcDAIn/mw8A+Nh23nnnLFmyJL/61a8aLR87dmwqKioa3QV4jTXWWGqEbNGiRZOzrS644IJGX8tsTi1atMiee+6Z3//+93niiSearH/ttddWuI0+ffpk/fXXzy9+8Yu8/fbbq7SND9t9991TWVmZ0aNHNzkLrOH9HDhwYGpqanL66acv9fqLK9pvQwD6LOLxHnvskRYtWuTkk09u8nkolUqZO3fucl/fr1+/JMnf/va3T3Reb775ZpP5NATiRYsWNVo+bdq01NbWNvra9spq0aJFBgwYkD/96U+ZNWtWefmcOXNyzTXXZKuttlrlr/Dvs88+efnll3PZZZc1WbdgwYLyHcYbzob88GUZxo0b1+g1AwYMSJs2bTJmzJgsXLiw0brmOHPy/fffzyWXXFJ+vnjx4lxyySVp3759+vTps8LXL+130PXXX1++junKatmyZfku5ePHj88mm2zyqcRzAPhv5gxLAOBj23XXXbPddtvlpz/9aWbNmpWvfvWrueOOO/KnP/0pI0aMKJ+Jlfw72k2ePDnnnntuOnfunG7duqVv377ZZZddcvXVV6e2tja9evXK1KlTM3ny5KVeO665nHHGGZkyZUr69u2bQw89NL169cobb7yRv//975k8eXLeeOON5b6+srIyl19+eXbaaaf07t07Bx98cNZZZ528/PLLmTJlSmpqanLzzTd/pDltsMEG+elPf5pTTjklW2+9dfbYY49UV1fn4YcfTufOnTNmzJjU1NTkoosuygEHHJDNNtssgwcPTvv27fPiiy/m1ltvzZZbbtkkNn/QpptumhYtWuTMM8/M/PnzU11dne233z4dOnT4SHNdGeuvv35OPfXUnHDCCZk1a1Z23333tGnTJjNnzswf//jHHHbYYTnmmGOW+fru3bvny1/+ciZPntzkBj8fx5VXXpkLL7ww3/3ud7P++uvnrbfeymWXXZaamprsvPPOjcZOmjQpu+666yqfmXrqqadm0qRJ2WqrrXLkkUemZcuWueSSS7Jo0aKcddZZq3wMBxxwQK677rocfvjhmTJlSrbccsssWbIkTz/9dK677rpMnDgxX//61zNgwIBUVVVl1113zfe///28/fbbueyyy9KhQ4f861//Km+vpqYmY8eOzSGHHJJvfOMb2W+//bLWWmvlsccey7vvvpsrr7xylee6Kjp37pwzzzwzs2bNykYbbZRrr702jz76aC699NKVuvbpLrvsktGjR+fggw/ON7/5zTz++OOZMGFCo5sfrawDDzww559/fqZMmZIzzzxzVQ4HAD7XBEsA4GOrrKzMTTfdlFGjRuXaa6/NuHHjst566+Xss8/O0Ucf3Wjsueeem8MOOywnnnhiFixYkCFDhqRv374577zz0qJFi0yYMCELFy7MlltumcmTJ2fgwIHNdFRNdezYMQ899FBGjx6dP/zhD7nwwgvTrl279O7de6WjxLbbbpupU6fmlFNOya9+9au8/fbb6dSpU/r27dvoDscfxejRo9OtW7dccMEF+elPf5rWrVvnK1/5SqOvoe63337p3LlzzjjjjJx99tlZtGhR1llnnWy99dY5+OCDl7v9Tp065eKLL86YMWMybNiwLFmyJFOmTPlUgmWSHH/88dloo40yduzYnHzyyUn+fX3FAQMGZLfddlvh64cOHZpRo0ZlwYIFK319xxXp379/Hnroofzud7/LnDlzUltbm8033zwTJkxodPOVp59+Ok888UR++ctfrvK+evfunXvuuScnnHBCxowZk/r6+vTt2ze/+c1vync1XxWVlZW58cYbM3bs2Fx11VX54x//mNatW6d79+75n//5n/JX8DfeeOPccMMNOfHEE3PMMcekU6dOOeKII9K+ffsmEXjYsGHp0KFDzjjjjJxyyilZbbXV0qNHj4wcOXKV57mq1lprrVx55ZX54Q9/mMsuuywdO3bMr371qxx66KEr9fqf/OQneeedd3LNNdfk2muvzWabbZZbb701xx9//EeeS58+fdK7d+889dRT2X///T/y6wHg866i1FxXugYAgE/B/Pnz071795x11lkZNmzYZ7rvESNG5O677860adM+02t/Ujxf+9rX8oUvfCF/+ctfmnsqAPAfxzUsAQD4r1JbW5tjjz02Z5999id6h/kVmTt3bi6//PKceuqpYuXn3N/+9rc8+uijOfDAA5t7KgDwH8kZlgAAAJ+AJ554ItOmTcs555yT119/Pc8//3xWX3315p4WAPzHcYYlAADAJ+CGG27IwQcfnPfeey+//e1vxUoAWEXOsAQAAAAACsMZlgAAAABAYQiWAAAAAEBhtGzuCfwnqK+vzyuvvJI2bdq44yMAAAAAfESlUilvvfVWOnfunMrK5Z9DKViuhFdeeSVdunRp7mkAAAAAwH+0l156Keuuu+5yxwiWK6FNmzZJ/v2G1tTUNPNsAAAAAOA/S11dXbp06VLubMsjWK6Ehq+B19TUCJYAAAAAsIpW5nKLbroDAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFEbL5p4AxbHe8bc29xQAmtWsMwY19xQAAAA+95xhCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFEazBsv11lsvFRUVTR7Dhw9PkixcuDDDhw9Pu3btsuaaa2bPPffMnDlzGm3jxRdfzKBBg9K6det06NAhP/7xj/P+++83GnPnnXdms802S3V1dTbYYIOMHz/+szpEAAAAAOAjaNZg+fDDD+df//pX+TFp0qQkyd57750kGTlyZG6++eZcf/31ueuuu/LKK69kjz32KL9+yZIlGTRoUBYvXpz7778/V155ZcaPH59Ro0aVx8ycOTODBg3Kdtttl0cffTQjRozIIYcckokTJ362BwsAAAAArFBFqVQqNfckGowYMSK33HJLZsyYkbq6urRv3z7XXHNN9tprryTJ008/nZ49e2bq1KnZYostctttt2WXXXbJK6+8ko4dOyZJLr744hx33HF57bXXUlVVleOOOy633nprnnjiifJ+Bg8enHnz5uX2229fqXnV1dWltrY28+fPT01NzSd/4AWx3vG3NvcUAJrVrDMGNfcUAAAA/it9lL5WmGtYLl68OL/5zW8ydOjQVFRUZNq0aXnvvfey4447lsf06NEjX/rSlzJ16tQkydSpU7PJJpuUY2WSDBw4MHV1dXnyySfLYz64jYYxDdtYmkWLFqWurq7RAwAAAAD49BUmWN54442ZN29eDjrooCTJ7NmzU1VVlbZt2zYa17Fjx8yePbs85oOxsmF9w7rljamrq8uCBQuWOpcxY8aktra2/OjSpcvHPj4AAAAAYMUKEyx//etfZ6eddkrnzp2beyo54YQTMn/+/PLjpZdeau4pAQAAAMDnQsvmnkCSvPDCC5k8eXL+8Ic/lJd16tQpixcvzrx58xqdZTlnzpx06tSpPOahhx5qtK2Gu4h/cMyH7yw+Z86c1NTUpFWrVkudT3V1daqrqz/+gQEAAAAAH0khzrAcN25cOnTokEGD/u9mB3369Mlqq62Wv/zlL+VlzzzzTF588cX069cvSdKvX788/vjjefXVV8tjJk2alJqamvTq1as85oPbaBjTsA0AAAAAoDiaPVjW19dn3LhxGTJkSFq2/L8TPmtrazNs2LAcddRRmTJlSqZNm5aDDz44/fr1yxZbbJEkGTBgQHr16pUDDjggjz32WCZOnJgTTzwxw4cPL58hefjhh+f555/Psccem6effjoXXnhhrrvuuowcObJZjhcAAAAAWLZm/0r45MmT8+KLL2bo0KFN1o0dOzaVlZXZc889s2jRogwcODAXXnhheX2LFi1yyy235Igjjki/fv2yxhprZMiQIRk9enR5TLdu3XLrrbdm5MiROe+887Luuuvm8ssvz8CBAz+T4wMAAAAAVl5FqVQqNfckiq6uri61tbWZP39+ampqmns6n5r1jr+1uacA0KxmnTFoxYMAAAD4yD5KX2v2r4QDAAAAADQQLAEAAACAwhAsAQAAAIDCECwBAIJM8s4AACAASURBVAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwmjZ3BMAAP47rHf8rc09BYBmNeuMQc09BQD4r+AMSwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDCaPVi+/PLL+d73vpd27dqlVatW2WSTTfK3v/2tvL5UKmXUqFFZe+2106pVq+y4446ZMWNGo2288cYb2X///VNTU5O2bdtm2LBhefvttxuN+cc//pGtt946q6++erp06ZKzzjrrMzk+AAAAAGDlNWuwfPPNN7PllltmtdVWy2233Zbp06fnnHPOyVprrVUec9ZZZ+X888/PxRdfnAcffDBrrLFGBg4cmIULF5bH7L///nnyySczadKk3HLLLbn77rtz2GGHldfX1dVlwIAB6dq1a6ZNm5azzz47J510Ui699NLP9HgBAAAAgOVr2Zw7P/PMM9OlS5eMGzeuvKxbt27lP5dKpfzyl7/MiSeemO985ztJkquuuiodO3bMjTfemMGDB+epp57K7bffnocffjhf//rXkyQXXHBBdt555/ziF79I586dM2HChCxevDhXXHFFqqqq0rt37zz66KM599xzG4VNAAAAAKB5NesZljfddFO+/vWvZ++9906HDh3yta99LZdddll5/cyZMzN79uzsuOOO5WW1tbXp27dvpk6dmiSZOnVq2rZtW46VSbLjjjumsrIyDz74YHnMNttsk6qqqvKYgQMH5plnnsmbb77ZZF6LFi1KXV1dowcAAAAA8Olr1mD5/PPP56KLLsqGG26YiRMn5ogjjsiPfvSjXHnllUmS2bNnJ0k6duzY6HUdO3Ysr5s9e3Y6dOjQaH3Lli3zhS98odGYpW3jg/v4oDFjxqS2trb86NKlyydwtAAAAADAijRrsKyvr89mm22W008/PV/72tdy2GGH5dBDD83FF1/cnNPKCSeckPnz55cfL730UrPOBwAAAAA+L5o1WK699trp1atXo2U9e/bMiy++mCTp1KlTkmTOnDmNxsyZM6e8rlOnTnn11VcbrX///ffzxhtvNBqztG18cB8fVF1dnZqamkYPAAAAAODT16zBcsstt8wzzzzTaNk///nPdO3aNcm/b8DTqVOn/OUvfymvr6ury4MPPph+/folSfr165d58+Zl2rRp5TF//etfU19fn759+5bH3H333XnvvffKYyZNmpSNN9640R3JAQAAAIDm1azBcuTIkXnggQdy+umn59lnn80111yTSy+9NMOHD0+SVFRUZMSIETn11FNz00035fHHH8+BBx6Yzp07Z/fdd0/y7zMyv/3tb+fQQw/NQw89lPvuuy8/+MEPMnjw4HTu3DlJst9++6WqqirDhg3Lk08+mWuvvTbnnXdejjrqqGY7dgAAAACgqZbNufNvfOMb+eMf/5gTTjgho0ePTrdu3fLLX/4y+++/f3nMsccem3feeSeHHXZY5s2bl6222iq33357Vl999fKYCRMm5Ac/+EF22GGHVFZWZs8998z5559fXl9bW5s77rgjw4cPT58+ffLFL34xo0aNymGHHfaZHi8AAAAAsHwVpVKp1NyTKLq6urrU1tZm/vz5/9XXs1zv+FubewoAzWrWGYOaewr/0fw7Anze+XcEAJbto/S1Zv1KOAAAAADABwmWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGE0a7A86aSTUlFR0ejRo0eP8vqFCxdm+PDhadeuXdZcc83sueeemTNnTqNtvPjiixk0aFBat26dDh065Mc//nHef//9RmPuvPPObLbZZqmurs4GG2yQ8ePHfxaHBwAAAAB8RM1+hmXv3r3zr3/9q/y49957y+tGjhyZm2++Oddff33uuuuuvPLKK9ljjz3K65csWZJBgwZl8eLFuf/++3PllVdm/PjxGTVqVHnMzJkzM2jQoGy33XZ59NFHM2LEiBxyyCGZOHHiZ3qcAAAAAMCKtWz2CbRsmU6dOjVZPn/+/Pz617/ONddck+233z5JMm7cuPTs2TMPPPBAtthii9xxxx2ZPn16Jk+enI4dO2bTTTfNKaeckuOOOy4nnXRSqqqqcvHFF6dbt24555xzkiQ9e/bMvffem7Fjx2bgwIGf6bECAAAAAMvX7GdYzpgxI507d0737t2z//7758UXX0ySTJs2Le+991523HHH8tgePXrkS1/6UqZOnZokmTp1ajbZZJN07NixPGbgwIGpq6vLk08+WR7zwW00jGnYxtIsWrQodXV1jR4AAAAAwKevWYNl3759M378+Nx+++256KKLMnPmzGy99dZ56623Mnv27FRVVaVt27aNXtOxY8fMnj07STJ79uxGsbJhfcO65Y2pq6vLggULljqvMWPGpLa2tvzo0qXLJ3K8AAAAAMDyNetXwnfaaafyn7/yla+kb9++6dq1a6677rq0atWq2eZ1wgkn5Kijjio/r6urEy0BAAAA4DPQ7F8J/6C2bdtmo402yrPPPptOnTpl8eLFmTdvXqMxc+bMKV/zslOnTk3uGt7wfEVjampqlhlFq6urU1NT0+gBAAAAAHz6ChUs33777Tz33HNZe+2106dPn6y22mr5y1/+Ul7/zDPP5MUXX0y/fv2SJP369cvjjz+eV199tTxm0qRJqampSa9evcpjPriNhjEN2wAAAAAAiqNZg+UxxxyTu+66K7Nmzcr999+f7373u2nRokX23Xff1NbWZtiwYTnqqKMyZcqUTJs2LQcffHD69euXLbbYIkkyYMCA9OrVKwcccEAee+yxTJw4MSeeeGKGDx+e6urqJMnhhx+e559/Pscee2yefvrpXHjhhbnuuusycuTI5jx0AAAAAGApmvUalv/v//2/7Lvvvpk7d27at2+frbbaKg888EDat2+fJBk7dmwqKyuz5557ZtGiRRk4cGAuvPDC8utbtGiRW265JUcccUT69euXNdZYI0OGDMno0aPLY7p165Zbb701I0eOzHnnnZd11103l19+eQYOHPiZHy8AAAAAsHwVpVKp1NyTKLq6urrU1tZm/vz5/9XXs1zv+FubewoAzWrWGYOaewr/0fw7Anze+XcEAJbto/S1Ql3DEgAAAAD4fBMsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCECwBAAAAgMIQLAEAAACAwhAsAQAAAIDCaPlRXzBz5szcc889eeGFF/Luu++mffv2+drXvpZ+/fpl9dVX/zTmCAAAAAB8Tqx0sJwwYULO+//s3X2Ul3Wd//HXyM2AyoyiziCJxEamqKRCJydbb4gkpYiV9qwnvFtQs0VNaJUo1xIz0cKb0tIiQytXc7UyyRtExUK8CQPxZtVVdEwb0BRGCBlu5vfH7/A9jpg7X51xru37eJzzPYe5Pp+55n39xTnPc32v65JL8oc//CH19fXp379/evfunVdeeSVPP/10evXqlfHjx2fq1KkZOHBgZ84MAAAAAPydalew3HfffdOzZ88cd9xxueGGGzJgwIA26+vWrcvChQtz7bXXZvjw4fn+97+ff/7nf+6UgQEAAACAv1/tCpYzZszIqFGj/uZ6dXV1Dj744Bx88ME599xz8+yzz3bUfAAAAABABWlXsHy7WPlmO+ywQ3bYYYd3PBAAAAAAULnKfkv4Qw89lKVLl5Z+/vWvf52xY8fmq1/9alpaWjp0OAAAAACgspQdLL/whS/kySefTJI888wzOfLII7P11lvn+uuvzxlnnNHhAwIAAAAAlaPsYPnkk09mn332SZJcf/31OfDAA3PNNddk9uzZueGGGzp8QAAAAACgcpQdLFtbW7Np06YkyR133JHDDz88STJgwIC8/PLLHTsdAAAAAFBRyg6Ww4cPzze/+c389Kc/zfz58zN69OgkybJly1JfX9/hAwIAAAAAlaPsYHnxxRfnoYceysknn5yvfe1rGTx4cJLkv/7rv/Kxj32swwcEAAAAACpH93J/YejQoW3eEr7Zt7/97XTr1q1DhgIAAAAAKlPZwfKNVq9eXXqe5WY9evR4VwMBAAAAAJWr7K+EL1u2LKNHj84222yT2trabL/99tl+++2z3XbbZfvtt++MGQEAAACAClH2HZZHHXVUWltbc+WVV6a+vj5VVVWdMRcAAAAAUIHKDpZLlizJokWL8qEPfagz5gEAAAAAKljZXwn/yEc+kueff74zZgEAAAAAKlzZd1jOmjUrJ510Ul544YXstddeW7xkZ+jQoR02HAAAAABQWcoOli+99FKefvrp/Ou//mvpWFVVVVpbW1NVVZWNGzd26IAAAAAAQOUoO1hOmDAh++67b/7zP//TS3cAAAAAgA5VdrB87rnnctNNN2Xw4MGdMQ8AAAAAUMHKfunOiBEjsmTJks6YBQAAAACocGXfYfmZz3wmkydPztKlS7P33ntv8dKdMWPGdNhwAAAAAEBlKTtYnnTSSUmS6dOnb7HmpTsAAAAAwLtRdrDctGlTZ8wBAAAAAFD+MywBAAAAADpLu4Lltdde2+4TPv/881mwYME7HggAAAAAqFztCpY/+MEPsscee+SCCy7I448/vsX6qlWr8tvf/jaf//zns99+++Uvf/lLhw8KAAAAAPz9a9czLOfPn5+bbrop3/ve9zJt2rRss802qa+vT69evfLqq6+mqakpO+64Y4477rg88sgjqa+v7+y5AQAAAIC/Q+1+6c6YMWMyZsyYvPzyy/n973+f5557LmvXrs2OO+6YfffdN/vuu2+22sojMQEAAACAd67st4TvuOOOGTt2bGfMAgAAAABUOLdEAgAAAACFIVgCAAAAAIUhWAIAAAAAhSFYAgAAAACF8Y6DZUtLS5544ols2LChI+cBAAAAACpY2cHyr3/9ayZOnJitt946e+65ZxobG5Mkp5xySmbMmNHhAwIAAAAAlaPsYDlt2rQsWbIkd999d3r16lU6PnLkyFx33XUdOhwAAAAAUFm6l/sLv/rVr3Lddddl//33T1VVVen4nnvumaeffrpDhwMAAAAAKkvZd1i+9NJLqaur2+L4mjVr2gRMAAAAAIBylR0shw8fnjlz5pR+3hwpZ82alYaGho6bDAAAAACoOGV/Jfxb3/pWDjvssDz22GPZsGFDLrnkkjz22GO59957M3/+/M6YEQAAAACoEGXfYfnxj388ixcvzoYNG7L33nvn9ttvT11dXRYuXJhhw4Z1xowAAAAAQIUo+w7LJPnABz6QH/3oRx09CwAAAABQ4d5RsEySFStWZMWKFdm0aVOb40OHDn3XQwEAAAAAlansYLlo0aIce+yxefzxx9Pa2tpmraqqKhs3buyw4QAAAACAylJ2sJwwYUJ22223/PjHP059fX3pLeEAAAAAAO9W2cHymWeeyQ033JDBgwd3xjwAAAAAQAUr+y3hn/jEJ7JkyZLOmAUAAAAAqHBl32E5a9asHHvssXnkkUey1157pUePHm3Wx4wZ02HDAQAAAACVpexguXDhwixYsCC33HLLFmteugMAAAAAvBtlfyX8lFNOyVFHHZU///nP2bRpU5uPWAkAAAAAvBtlB8u//OUvmTx5curr6ztjHgAAAACggpUdLI844ojcddddnTELAAAAAFDhyn6G5W677ZZp06bl97//ffbee+8tXrpz6qmndthwAAAAAEBleUdvCd92220zf/78zJ8/v81aVVWVYAkAAAAAvGNlB8tly5Z1xhwAAAAAAOU/wxIAAAAAoLO06w7LKVOm5Jxzzsk222yTKVOmvO3eCy+8sEMGAwAAAAAqT7uC5R//+MesX7++9G8AAAAAgM7QrmB51113veW/AQAAAAA6UtnPsJwwYUJee+21LY6vWbMmEyZM6JChAAAAAIDKVHawvOqqq7J27dotjq9duzZXX311hwwFAAAAAFSmdn0lPEmam5vT2tqa1tbWvPbaa+nVq1dpbePGjfntb3+burq6ThkSAAAAAKgM7Q6W2223XaqqqlJVVZXddttti/WqqqqcffbZHTocAAAAAFBZ2h0s77rrrrS2tmbEiBG54YYb0rdv39Jaz549M3DgwPTv379ThgQAAAAAKkO7g+VBBx2UJFm2bFl23XXXVFVVddpQAAAAAEBlKvulOwMHDuyUWDljxoxUVVXltNNOKx17/fXXM2nSpOywww7ZdtttM27cuCxfvrzN7zU2Nmb06NHZeuutU1dXl9NPPz0bNmxos+fuu+/Ofvvtl+rq6gwePDizZ8/u8PkBAAAAgHev7GDZGR588MFcccUVGTp0aJvjkydPzm9+85tcf/31mT9/fl588cUcccQRpfWNGzdm9OjRaWlpyb333purrroqs2fPzllnnVXas2zZsowePTqHHHJIFi9enNNOOy3HH398brvttvfs+gAAAACA9unyYLl69eqMHz8+P/rRj7L99tuXjq9atSo//vGPc+GFF2bEiBEZNmxYfvKTn+Tee+/NfffdlyS5/fbb89hjj+VnP/tZ9tlnnxx22GE555xzctlll6WlpSVJcvnll2fQoEGZOXNm9thjj5x88sn53Oc+l4suuqhLrhcAAAAA+Nu6PFhOmjQpo0ePzsiRI9scX7RoUdavX9/m+O67755dd901CxcuTJIsXLgwe++9d+rr60t7Ro0alebm5jz66KOlPW8+96hRo0rneCvr1q1Lc3Nzmw8AAAAA0PneUbDcsGFD7rjjjlxxxRV57bXXkiQvvvhiVq9eXdZ5rr322jz00EM577zztlhrampKz549s91227U5Xl9fn6amptKeN8bKzeub195uT3Nzc9auXfuWc5133nmpra0tfQYMGFDWdQEAAAAA70y73xK+2XPPPZdPfepTaWxszLp16/LJT34yffr0yfnnn59169bl8ssvb9d5nn/++XzpS1/K3Llz06tXr7IH70zTpk3LlClTSj83NzeLlgAAAADwHij7DssvfelLGT58eF599dX07t27dPyf/umfMm/evHafZ9GiRVmxYkX222+/dO/ePd27d8/8+fPz3e9+N927d099fX1aWlqycuXKNr+3fPny9OvXL0nSr1+/Ld4avvnn/21PTU1Nm/nfqLq6OjU1NW0+AAAAAEDnKztY/u53v8uZZ56Znj17tjn+/ve/Py+88EK7z/OJT3wiS5cuzeLFi0uf4cOHZ/z48aV/9+jRo00EfeKJJ9LY2JiGhoYkSUNDQ5YuXZoVK1aU9sydOzc1NTUZMmRIac+bQ+rcuXNL5wAAAAAAiqPsr4Rv2rQpGzdu3OL4n/70p/Tp06fd5+nTp0/22muvNse22Wab7LDDDqXjEydOzJQpU9K3b9/U1NTklFNOSUNDQ/bff/8kyaGHHpohQ4bk6KOPzgUXXJCmpqaceeaZmTRpUqqrq5MkJ510Ui699NKcccYZmTBhQu6888784he/yJw5c8q9dAAAAACgk5V9h+Whhx6aiy++uPRzVVVVVq9ena9//es5/PDDO3S4iy66KJ/+9Kczbty4HHjggenXr19uvPHG0nq3bt1y8803p1u3bmloaMhRRx2VY445JtOnTy/tGTRoUObMmZO5c+fmwx/+cGbOnJlZs2Zl1KhRHTorAAAAAPDuVbW2traW8wt/+tOfMmrUqLS2tuapp57K8OHD89RTT2XHHXfMPffck7q6us6atcs0NzentrY2q1at+rt+nuX7v+KuU6CyPTtjdFeP8H+a/0eASuf/EQD428rpa2V/JXyXXXbJkiVLct1112XJkiVZvXp1Jk6cmPHjx//Nl9gAAAAAALRH2cEySbp3757x48dn/PjxHT0PAAAAAFDByn6G5XnnnZcrr7xyi+NXXnllzj///A4ZCgAAAACoTGUHyyuuuCK77777Fsf33HPPXH755R0yFAAAAABQmcoOlk1NTdl55523OL7TTjvlz3/+c4cMBQAAAABUprKD5YABA7JgwYItji9YsCD9+/fvkKEAAAAAgMpU9kt3TjjhhJx22mlZv359RowYkSSZN29ezjjjjHz5y1/u8AEBAAAAgMpRdrA8/fTT85e//CX/9m//lpaWliRJr169MnXq1EybNq3DBwQAAAAAKkfZwbKqqirnn39+/uM//iOPP/54evfunQ9+8IOprq7ujPkAAAAAgApSdrDcbNttt81HPvKRjpwFAAAAAKhwZQfLNWvWZMaMGZk3b15WrFiRTZs2tVl/5plnOmw4AAAAAKCylB0sjz/++MyfPz9HH310dt5551RVVXXGXAAAAABABSo7WN5yyy2ZM2dODjjggM6YBwAAAACoYFuV+wvbb799+vbt2xmzAAAAAAAVruxgec455+Sss87KX//6186YBwAAAACoYGV/JXzmzJl5+umnU19fn/e///3p0aNHm/WHHnqow4YDAAAAACpL2cFy7NixnTEHAAAAAED5wfLrX/96Z8wBAAAAAFD+MyyTZOXKlZk1a1amTZuWV155Jcn//yr4Cy+80KHDAQAAAACVkcr9RAAAIABJREFUpew7LB9++OGMHDkytbW1efbZZ3PCCSekb9++ufHGG9PY2Jirr766M+YEAAAAACpA2XdYTpkyJccdd1yeeuqp9OrVq3T88MMPzz333NOhwwEAAAAAlaXsYPnggw/mC1/4whbH3/e+96WpqalDhgIAAAAAKlPZwbK6ujrNzc1bHH/yySez0047dchQAAAAAEBlKjtYjhkzJtOnT8/69euTJFVVVWlsbMzUqVMzbty4Dh8QAAAAAKgcZQfLmTNnZvXq1amrq8vatWtz0EEHZfDgwenTp0/OPffczpgRAAAAAKgQZb8lvLa2NnPnzs2CBQuyZMmSrF69Ovvtt19GjhzZGfMBAAAAABWkrGC5fv369O7dO4sXL84BBxyQAw44oLPmAgAAAAAqUFlfCe/Ro0d23XXXbNy4sbPmAQAAAAAqWNnPsPza176Wr371q3nllVc6Yx4AAAAAoIKV/QzLSy+9NP/zP/+T/v37Z+DAgdlmm23arD/00EMdNhwAAAAAUFnKDpZjx47tjDkAAAAAAMoPll//+tc7Yw4AAAAAgPKfYZkkK1euzKxZszJt2rTSsywfeuihvPDCCx06HAAAAABQWcq+w/Lhhx/OyJEjU1tbm2effTYnnHBC+vbtmxtvvDGNjY25+uqrO2NOAAAAAKAClH2H5ZQpU3LcccflqaeeSq9evUrHDz/88Nxzzz0dOhwAAAAAUFnKDpYPPvhgvvCFL2xx/H3ve1+ampo6ZCgAAAAAoDKVHSyrq6vT3Ny8xfEnn3wyO+20U4cMBQAAAABUprKD5ZgxYzJ9+vSsX78+SVJVVZXGxsZMnTo148aN6/ABAQAAAIDKUXawnDlzZlavXp26urqsXbs2Bx10UAYPHpw+ffrk3HPP7YwZAQAAAIAKUfZbwmtrazN37twsWLAgS5YsyerVq7Pffvtl5MiRnTEfAAAAAFBB2hUs+/btmyeffDI77rhjJkyYkEsuuSQHHHBADjjggM6eDwAAAACoIO36SnhLS0vpRTtXXXVVXn/99U4dCgAAAACoTO26w7KhoSFjx47NsGHD0tramlNPPTW9e/d+y71XXnllhw4IAAAAAFSOdgXLn/3sZ7nooovy9NNPp6qqKqtWrXKXJQAAAADQ4doVLOvr6zNjxowkyaBBg/LTn/40O+ywQ6cOBgAAAABUnrLfEr5s2bLOmAMAAAAAoPxgmSTz5s3LvHnzsmLFimzatKnNmmdYAgAAAADvVNnB8uyzz8706dMzfPjw7LzzzqmqquqMuQAAAACAClR2sLz88ssze/bsHH300Z0xDwAAAABQwbYq9xdaWlrysY99rDNmAQAAAAAqXNnB8vjjj88111zTGbMAAAAAABWu7K+Ev/766/nhD3+YO+64I0OHDk2PHj3arF944YUdNhwAAAAAUFnKDpYPP/xw9tlnnyTJI4880mbNC3gAAAAAgHej7GB51113dcYcAAAAAADlP8MSAAAAAKCztPsOyyOOOKJd+2688cZ3PAwAAAAAUNnaHSxra2s7cw4AAAAAgPYHy5/85CedOQcAAAAAgGdYAgAAAADFIVgCAAAAAIUhWAIAAAAAhSFYAgAAAACFIVgCAAAAAIXRrreE33TTTe0+4ZgxY97xMAAAAABAZWtXsBw7dmy7TlZVVZWNGze+q4EAAAAAgMrVrmC5adOmzp4DAAAAAMAzLAEAAACA4mjXHZZvtmbNmsyfPz+NjY1paWlps3bqqad2yGAAAAAAQOUpO1j+8Y9/zOGHH56//vWvWbNmTfr27ZuXX345W2+9derq6gRLAAAAAOAdK/sr4ZMnT85nPvOZvPrqq+ndu3fuu+++PPfccxk2bFi+853vdMaMAAAAAECFKDtYLl68OF/+8pez1VZbpVu3blm3bl0GDBiQCy64IF/96lc7Y0YAAAAAoEKUHSx79OiRrbb6/79WV1eXxsbGJEltbW2ef/75jp0OAAAAAKgoZT/Dct99982DDz6YD37wgznooINy1lln5eWXX85Pf/rT7LXXXp0xIwAAAABQIcq+w/Jb3/pWdt555yTJueeem+233z5f/OIX89JLL+WKK67o8AEBAAAAgMpR9h2Ww4cPL/27rq4ut956a4cOBAAAAABUrrLvsBwxYkRWrly5xfHm5uaMGDGiQ4YCAAAAACpT2cHy7rvvTktLyxbHX3/99fzud7/rkKEAAAAAgMrU7q+EP/zww6V/P/bYY2lqair9vHHjxtx666153/ve17HTAQAAAAAVpd3Bcp999klVVVWqqqre8qvfvXv3zve+970OHQ4AAAAAqCztDpbLli1La2tr/uEf/iEPPPBAdtppp9Jaz549U1dXl27dunXKkAAAAABAZWh3sBw4cGCSZNOmTZ02DAAAAABQ2dodLN/o6aefzsUXX5zHH388STJkyJB86Utfygc+8IEOHQ4AAAAAqCxlvyX8tttuy5AhQ/LAAw9k6NChGTp0aO6///7sueeemTt3bmfMCAAAAABUiLLvsPzKV76SyZMnZ8aMGVscnzp1aj75yU922HAAAAAAQGUp+w7Lxx9/PBMnTtzi+IQJE/LYY491yFAAAAAAQGUqO1jutNNOWbx48RbHFy9enLq6ug4ZCgAAAACoTO3+Svj06dPz7//+7znhhBNy4okn5plnnsnHPvaxJMmCBQty/vnnZ8qUKZ02KAAAAADw96/dwfLss8/OSSedlP/4j/9Inz59MnPmzEybNi1J0r9//3zjG9/Iqaee2mmDAgAAAAB//9odLFtbW5MkVVVVmTx5ciZPnpzXXnstSdKnT5/OmQ4AAAAAqChlvSW8qqqqzc9CJQAAAADQkcoKlrvtttsW0fLNXnnllXc1EAAAAABQucoKlmeffXZqa2s7axYAAAAAoMKVFSyPPPLI1NXVddYsAAAAAECF26q9G/+3r4IDAAAAALxb7Q6Wm98SDgAAAADQWdr9lfBNmzZ15hwAAAAAAO2/wxIAAAAAoLMJlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhdGmw/MEPfpChQ4empqYmNTU1aWhoyC233FJaf/311zNp0qTssMMO2XbbbTNu3LgsX768zTkaGxszevTobL311qmrq8vpp5+eDRs2tNlz9913Z7/99kt1dXUGDx6c2bNnvxeXBwAAAACUqUuD5S677JIZM2Zk0aJF+cMf/pARI0bks5/9bB599NEkyeTJk/Ob3/wm119/febPn58XX3wxRxxxROn3N27cmNGjR6elpSX33ntvrrrqqsyePTtnnXVWac+yZcsyevToHHLIIVm8eHFOO+20HH/88bntttve8+sFAAAAAN5eVWtra2tXD/FGffv2zbe//e187nOfy0477ZRrrrkmn/vc55Ik//3f/5099tgjCxcuzP77759bbrkln/70p/Piiy+mvr4+SXL55Zdn6tSpeemll9KzZ89MnTo1c+bMySOPPFL6G0ceeWRWrlyZW2+9tV0zNTc3p7a2NqtWrUpNTU3HX3RBvP8rc7p6BIAu9eyM0V09wv9p/h8BKp3/RwDgbyunrxXmGZYbN27MtddemzVr1qShoSGLFi3K+vXrM3LkyNKe3XffPbvuumsWLlyYJFm4cGH23nvvUqxMklGjRqW5ubl0l+bChQvbnGPzns3neCvr1q1Lc3Nzmw8AAAAA0Pm6PFguXbo02267baqrq3PSSSfll7/8ZYYMGZKmpqb07Nkz2223XZv99fX1aWpqSpI0NTW1iZWb1zevvd2e5ubmrF279i1nOu+881JbW1v6DBgwoEOuFQAAAAB4e10eLD/0oQ9l8eLFuf/++/PFL34xxx57bB577LEunWnatGlZtWpV6fP888936TwAAAAAUCm6d/UAPXv2zODBg5Mkw4YNy4MPPphLLrkk//Iv/5KWlpasXLmyzV2Wy5cvT79+/ZIk/fr1ywMPPNDmfJvfIv7GPW9+s/jy5ctTU1OT3r17v+VM1dXVqa6u7pgLBAAAAADarcvvsHyzTZs2Zd26dRk2bFh69OiRefPmldaeeOKJNDY2pqGhIUnS0NCQpUuXZsWKFaU9c+fOTU1NTYYMGVLa88ZzbN6z+RwAAAAAQHF06R2W06ZNy2GHHZZdd901r732Wq655prcfffdue2221JbW5uJEydmypQp6du3b2pqanLKKaekoaEh+++/f5Lk0EMPzZAhQ3L00UfnggsuSFNTU84888xMmjSpdIfkSSedlEsvvTRnnHFGJkyYkDvvvDO/+MUvMmeON5kCAAAAQNF0abBcsWJFjjnmmPz5z39ObW1thg4dmttuuy2f/OQnkyQXXXRRttpqq4wbNy7r1q3LqFGj8v3vf7/0+926dcvNN9+cL37xi2loaMg222yTY489NtOnTy/tGTRoUObMmZPJkyfnkksuyS677JJZs2Zl1KhR7/n1AgAAAABvr6q1tbW1q4couubm5tTW1mbVqlWpqanp6nE6zfu/4q5ToLI9O2N0V4/wf5r/R4BK5/8RAPjbyulrhXuGJQAAAABQuQRLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMARLAAAAAKAwBEsAAAAAoDAESwAAAACgMLp39QAAAACQJO//ypyuHgGgSz07Y3RXj1AI7rAEAAAAAApDsAQAAAAACkOwBAAAAAAKQ7AEAAAAAApDsAQAAAAACkOwBAAAAAAKQ7AEAAAAAApDsAQAAAAACkOwBAAAAAAKQ7AEAAAAAApDsAQAAAAACkOwBAAAAAAKQ7AEAAAAAApDsAQAAAAACkOwBAAAAAAKQ7AEAAAAAAqjS4Pleeedl4985CPp06dP6urqMnbs2DzxxBNt9rz++uuZNGlSdthhh2y77bYZN25cli9f3mZPY2NjRo8ena233jp1dXU5/fTTs2HDhjZ77r777uy3336prq7O4MGDM3v27M6+PAAAAACgTF0aLOfPn59Jkyblvvvuy9y5c7N+/foceuihWbNmTWnP5MmT85vf/CbXX3995s+fnxdffDFHHHFEaX3jxo0ZPXp0Wlpacu+99+aqq67K7Nmzc9ZZZ5X2LFu2LKNHj84hhxySxYsX57TTTsvxxx+f22677T29XgAAAADg7XXvyj9+6623tvl59uzZqaury6JFi3LggQdm1apV+fGPf5xrrrkmI0aMSJL85Cc/yR577JH77rsv+++/f26//fY89thjueOOO1JfX5999tkn55xzTqZOnZpvfOMb6dmzZy6//PIMGjQoM2fOTJLsscce+f3vf5+LLrooo0aNes+vGwAAAAB4a4V6huWqVauSJH379k2SLFq0KOvXr8/IkSNLe3bffffsuuuuWbhwYZJk4cKF2XvvvVNfX1/aM2rUqDQ3N+fRRx8t7XnjOTbv2XyON1u3bl2am5vbfAAAAACAzleYYLlp06acdtppOeCAA7LXXnslSZqamtKzZ89st912bfbW19enqamptOeNsXLz+ua1t9vT3NyctWvXbjHLeeedl9ra2tJnwIABHXORAAAAAMDbKkywnDRpUh555JFce+21XT1Kpk2bllWrVpU+zz//fFePBAAAAAAVoUufYbnZySefnJtvvjn33HNPdtlll9Lxfv36paWlJStXrmxzl+Xy5cvTr1+/0p4HHnigzfk2v0X8jXve/Gbx5cuXp6amJr17995inurq6lRXV3fMxQEAAAAA7dald1i2trbm5JNPzi9/+cvceeedGTRoUJv1YcOGpUePHpk3b17p2BNPPJHGxsY0NDQkSRoaGrJ06dKsWLGitGfu3LmpqanJkCFDSnveeI7NezafAwAAAAAohi69w3LSpEm55ppr8utf/zp9+vQpPXOytrY2vXv3Tm1tbSZOnJgpU6akb9++qampySmnnJKGhobsv//+SZJDDz00Q4YMydFHH50LLrggTU1NOfPMMzNp0qTSXZInnXRSLr300pxxxhmZMGFC7rzzzvziF7/InDlzuuzaAQAAAIAtdekdlj/4wQ+yatWqHHzwwdl5551Ln+uuu66056KLLsqnP/3pjBs3LgceeGD69euXG2+8sbTerVu33HzzzenWrVsaGhpy1FFH5Zhjjsn06dNLewYNGpQ5c+Zk7ty5+fCHP5yZM2dm1qxZGTVq1Ht6vQAAAADA2+vSOyxbW1v/1z29evXKZZddlssuu+xv7hk4cGB++9vfvu15Dj744Pzxj38se0YAAAAA4L1TmLeEAwAAAAAIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhCJYAAAAAQGEIlgAAAABAYQiWAAAAAEBhdGmwvOeee/KZz3wm/fv3T1VVVX71q1+1WW9tbc1ZZ52VnXfeOb17987IkSPz1FNPtdnzyiuvZPz48ampqcl2222XiRMnZvXq1W32PPzww/nHf/zH9OrVKwMGDMgFF1zQ6dcGAAAAAJSvS4PlmjVr8uEPfziXXXbZW65fcMEF+e53v5vLL788999/f7bZZpuMGjUqr7/+emnP+PHj8+ijj2bu3Lm5+eabc8899+TEE08srTc3N+fQQw/NwIEDs2jRonz729/ON77xjfzwhz/s9OsDAAAAAMrTvSv/+GGHHZbDDjvsLddaW1tz8cUX58wzz8xnP/vZJMnVV1+d+vr6/OpXv8qRRx6Zxx9/PLfeemsefPDBDB8+PEnyve99L4cffni+853vpH///vn5z3+elpaWXHnllenZs2f23HPPLF68OBdeeGGbsAkAAAAAdL3CPsNy2bJlaWpqysiRI0vHamtr89GPfjQLFy5MkixcuDDbbbddKVYmyciRI7PVVlvl/vvvL+058MAD07Nnz9KeUaNG5Yknnsirr776ln973bp1aW5ubvMBAAAAADpfYYNlU1NTkqS+vr7N8fr6+tJaU1NT6urq2qx37949ffv2bbPnrc7xxr/xZuedd15qa2tLnwEDBrz7CwIAAAAA/leFDZZdadq0aVm1alXp8/zzz3f1SAAAAABQEQobLPv165ckWb58eZvjy5cvL63169cvK1asaLO+YcOGvPLKK232vNU53vg33qy6ujo1NTVtPgAAAABA5ytssBw0aFD69euXefPmlY41Nzfn/vvvT0NDQ5KkoaEhK1euzKJFi0p77rzzzmzatCkf/ehHS3vuueeerF+/vrRn7ty5+dCHPpTtt9/+PboaAAAAAKA9ujRYrl69OosXL87ixYuT/P8X7SxevDiNjY2pqqrKaaedlm9+85u56aabsnTp0hxzzDHp379/xo4dmyTZY4898qlPfSonnHBCHnjggSxYsCAnn3xyjjzyyPTv3z9J8vnPfz49e/bMxIkT8+ijj+a6667LJZdckilTpnTZdQMAAAAAb617V/7xP/zhDznkkENKP2+OiMcee2xmz56dM844I2vWrMmJJ56YlStX5uMf/3huvfXW9OrVq/Q7P//5z3PyySfnE5/4RLbaaquMGzcu3/3ud0vrtbW1uf322zNp0qQMGzYsO+64Y84666yceOKJ792FAgAAAADt0qXB8uCDD05ra+vfXK+qqsr06dMzffr0v7mnb9++ueaaa9727wwdOjS/+93v3vGcAAAAAMB7o7DPsAQAAAAAKo9gCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgCQAAAAAUhmAJAAAAABSGYAkAAAAAFIZgyf9r7+5jq6rvP4B/SksLVEBBoaViu40HBZUR0YV0G5ihbJptuBA2lcnDhgbL7ECIEJhUNnQ+oTIF4xS2KDASo4SpwymL4rNR6dwGIkRxTJ7mGkFwFOSe3x+u/XEtTyrYQ/t6JTfpPd/vOedzStIP933P/V4AAAAASA2BJQAAAACQGgJLAAAAACA1BJYAAAAAQGoILAEAAACA1BBYAgAAAACpIbAEAAAAAFJDYAkAAAAApIbAEgAAAABIDYElAAAAAJAaAksAAAAAIDUElgAAAABAaggsAQAAAIDUEFgCAAAAAKkhsAQAAAAAUkNgCQAAAACkhsASAAAAAEgNgSUAAAAAkBoCSwAAAAAgNQSWAAAAAEBqCCwBAAAAgNQQWAIAAAAAqSGwBAAAAABSQ2AJAAAAAKSGwBIAAAAASA2BJQAAAACQGgJLAAAAACA1BJYAAAAAQGoILAEAAACA1BBYAgAAAACpIbAEAAAAAFJDYAkAAAAApIbAEgAAAABIDYElAAAAAJAaAksAAAAAIDUElgAAAABAaggsAQAAAIDUEFgCAAAAAKkhsAQAAAAAUkNgCQAAAACkhsASAAAAAEgNgSUAAAAAkBoCSwAAAAAgNQSWAAAAAEBqCCwBAAAAgNQQWAIAAAAAqSGwBAAAAABSQ2AJAAAAAKSGwBIAAAAASA2BJQAAAACQGgJLAAAAACA1BJYAAAAAQGoILAEAAACA1BBYAgAAAACpIbAEAAAAAFJDYAkAAAAApIbAEgAAAABIDYElAAAAAJAaAksAAAAAIDUElgAAAABAaggsAQAAAIDUEFgCAAAAAKkhsAQAAAAAUkNgCQAAAACkhsASAAAAAEgNgSUAAAAAkBoCSwAAAAAgNQSWAAAAAEBqCCwBAAAAgNQQWAIAAAAAqSGwBAAAAABSQ2AJAAAAAKRwPLQ3AAAQKklEQVSGwBIAAAAASA2BJQAAAACQGs0qsLzrrruirKwsWrVqFV/72tfi5ZdfbuySAAAAAIB9NJvAcvHixTFhwoSYPn16vPbaa9GnT58YPHhwbN26tbFLAwAAAAD+p9kElrNmzYoxY8bEqFGjolevXnH33XdHmzZtYt68eY1dGgAAAADwP3mNXcAXYffu3fHqq6/GlClT6re1aNEiBg0aFC+88EKD+bW1tVFbW1v/fNu2bRERsX379qNfbCPK1H7Y2CUANKqm/nf+aNNHgOZOH/n89BKguWvKvaTu2pIkOeTcZhFYvvfee7F3797o3Llz1vbOnTvHG2+80WD+DTfcENddd12D7V27dj1qNQLQ+Nrf3tgVAHAs00cA+LyaQy/54IMPon379ged0ywCy09rypQpMWHChPrnmUwmampqomPHjpGTk9OIlUHTtX379ujatWts2LAh2rVr19jlAHCM0UcA+Lz0Eji6kiSJDz74ILp06XLIuc0isDzxxBMjNzc3tmzZkrV9y5YtUVRU1GB+QUFBFBQUZG07/vjjj2qNwMfatWvnPwcAfGb6CACfl14CR8+h7qys0yy+dCc/Pz/OOuusWL58ef22TCYTy5cvj/79+zdiZQAAAADAvprFHZYRERMmTIgRI0ZEv3794pxzzonbb789du7cGaNGjWrs0gAAAACA/8mtqqqqauwivginn356HH/88TFz5sy45ZZbIiJiwYIF0bNnz0auDKiTm5sbAwcOjLy8ZvNeCgBHkD4CwOell0A65CSH813iAAAAAABfgGaxhiUAAAAAcGwQWAIAAAAAqSGwBAAAAABSQ2AJpEJOTk4sWbKkscsA4BimlwAANA0CS2iiRo4cGTk5OZGTkxP5+fnRrVu3mDFjRnz00UeHtf/69esjJycnqqurj3Kln93AgQPrr3HfR9017jveqlWr6NWrV8yZM6d+/2effTbKy8ujY8eO0bp16zj11FPjtttuyzrHDTfcEGeffXa0bds2OnXqFEOGDIk1a9Z8odcJ0Fj0kkP3kk2bNsUll1wSPXr0iBYtWsTPf/7z/Z7n/fffj4qKiiguLo6CgoLo0aNHPPbYY1/INQKk2YYNG2L06NHRpUuXyM/Pj9LS0qisrIz//Oc/jV1aRGT3wn0f69atazC+v165a9euGDlyZJxxxhmRl5cXQ4YM2e95amtrY+rUqVFaWhoFBQVRVlYW8+bN+8KuE9JGYAlN2Le//e3YtGlTrF27Nq6++uqoqqqKm2+++QuvY8+ePUft2GPGjIlNmzZlPfLy8hqMr1q1KoYNGxYVFRWxaNGiiIgoLCyMcePGxYoVK2L16tUxbdq0mDZtWtxzzz31+z/99NNRUVERL774YjzxxBOxZ8+eOP/882Pnzp1H7ZoA0kQvOXgvqa2tjZNOOimmTZsWffr02e/xd+/eHeedd16sX78+HnzwwVizZk389re/jZKSkqN2TQDHgrfeeiv69esXa9eujUWLFsW6devi7rvvjuXLl0f//v2jpqamsUuMiP/vhfs+vvSlLzUY31+v3Lt3b7Ru3TquuuqqGDRo0AHPMWzYsFi+fHncd999sWbNmli0aFH07NnzqF8bpJXAEpqwgoKCKCoqitLS0hg7dmwMGjQoli5dGjt37ox27drFgw8+mDV/yZIlUVhYGB988EF9A+7bt2/k5OTEwIEDIyIik8nEjBkz4uSTT46CgoL46le/GsuWLas/Rt3dNIsXL44BAwZEq1atYsGCBRERMW/evOjdu3cUFBREcXFxjBs3Luv87733Xlx00UXRpk2b6N69eyxduvSQ19imTZsoKirKeuxv/Mtf/nJUVVVlHbdv375x8cUXR+/evaOsrCyGDx8egwcPjmeeeaZ+/2XLlsXIkSOjd+/e0adPn/jd734X//znP+PVV189zH8FgGObXnLwXlJWVhZ33HFHXHbZZdG+ffv9Hn/evHlRU1MTS5YsifLy8igrK4sBAwYcMOAEaC4qKioiPz8//vznP8eAAQPilFNOie985zvx5JNPxrvvvhtTp06tn1tWVhbXX399jB49Otq2bRunnHJK1o0Gdb3joYceinPPPTfatGkTffr0iRdeeKF+zujRo+PMM8+M2traiPj4DaW+ffvGZZdddtA663rhvo/c3NwG45/slREf3yQxd+7cGDNmTIP+UmfZsmXx9NNPx2OPPRaDBg2KsrKy6N+/f5SXl3/6Xyo0EQJLaEZat24du3fvjsLCwvjRj34U8+fPzxqfP39+DB06NNq2bRsvv/xyREQ8+eSTsWnTpnjooYciIuKOO+6IW2+9NW655ZZ4/fXXY/DgwfG9730v1q5dm3WsyZMnR2VlZaxevToGDx4cc+fOjYqKirj88svjb3/7WyxdujS6deuWtc91110Xw4YNi9dffz0uuOCCuPTSS4/4u6p1v4P9WblyZTz//PMxYMCAA+6/bdu2iIjo0KHDEa0L4Fihlxy8l+zP0qVLo3///lFRURGdO3eO008/Pa6//vrYu3fvEa0L4FhSU1MTjz/+eFx55ZXRunXrrLGioqK49NJLY/HixZEkSf32W2+9Nfr16xcrV66MK6+8MsaOHdtguaapU6fGxIkTo7q6Onr06BEXX3xx/cezZ8+eHTt37ozJkyfXz33//ffjzjvvPKLX9ln6RL9+/eKmm26KkpKS6NGjR0ycODH++9//HtG64JiSAE3SiBEjku9///tJkiRJJpNJnnjiiaSgoCCZOHFikiRJ8tJLLyW5ubnJxo0bkyRJki1btiR5eXnJU089lSRJkrz99ttJRCQrV67MOm6XLl2SmTNnZm07++yzkyuvvDJrv9tvv73BflOnTj1gvRGRTJs2rf75jh07kohI/vSnPx1wnwEDBiQtW7ZMCgsL6x8TJkzIGq+srEySJEk++uij5P77708iIrnzzjuzjlNSUpLk5+cnLVq0SGbMmHHA8+3duze58MILk/Ly8gPOAWhK9JLD7yWfnLuvnj17JgUFBcno0aOTV155JfnDH/6QdOjQIamqqjpgXQBN3YsvvphERPLwww/vd3zWrFlJRCRbtmxJkiRJSktLk+HDh9ePZzKZpFOnTsncuXOTJPn/3nHvvffWz/nHP/6RRESyevXq+m3PP/980rJly+QXv/hFkpeXlzzzzDMHrXPEiBFJbm5uVp8YOnRo1vjBeuUnj1U3d1+DBw9OCgoKkgsvvDB56aWXkkcffTQpLS1NRo4cedDaoCnL21+ICTQNjzzySBx33HGxZ8+eyGQycckll0RVVVVERJxzzjnRu3fv+P3vfx+TJ0+OBx54IEpLS+Ob3/zmAY+3ffv22LhxY4OPJpSXl8df//rXrG39+vWr/3nr1q2xcePG+Na3vnXQes8888z6nwsLC6Ndu3axdevWg+5z6aWXZn1U5Pjjj88anzNnTtx7772xe/fuyM3NjfHjx8fYsWOz5jzzzDOxY8eOePHFF2Py5MnRrVu3uPjiixucq6KiIv7+97/Hs88+e9CaAJoSveTwesnBZDKZ6NSpU9xzzz2Rm5sbZ511Vrz77rtx8803x/Tp0w/7OABNUbLPHZSHsu/f+JycnCgqKmrwN37fOcXFxRHxcQ859dRTIyKif//+MXHixPjlL38Z11xzTXz9618/5HnPPffcmDt3bv3zwsLCrPGD9crDkclkIicnJxYsWFC/vMisWbNi6NChMWfOnAZ3oEJzILCEJqyusebn50eXLl2yvkAgIuKnP/1p3HXXXTF58uSYP39+jBo1KnJyco7Iufdt4ofbYFu2bJn1PCcnJzKZzEH3ad++fYOPA+6r7kVo69ato7i4OFq0aLgSRt0aa2eccUZs2bIlqqqqGgSW48aNi0ceeSRWrFgRJ5988mFdD0BToJccXi85mOLi4mjZsmXWemennXZabN68OXbv3h35+fmf6ngATUG3bt0iJycnVq9eHRdddFGD8dWrV8cJJ5wQJ510Uv22w/kbv++cun6075xMJhPPPfdc5Obm1n/T96EUFhYetE8cqlceSnFxcZSUlGSthXzaaadFkiTxr3/9K7p37/6pjgdNgTUsoQmra6ynnHLKfpvm8OHD45133onZs2fHqlWrYsSIEfVjdS+e9l1fq127dtGlS5d47rnnso7z3HPPRa9evQ5YR9u2baOsrCyWL1/+eS/pU6t7EVpSUnJYLzAzmUz9ItwRH7/jO27cuHj44YfjL3/5S9a3AQI0B3rJp+8ln1ReXh7r1q3LesH85ptvRnFxsbASaLY6duwY5513XsyZM6fBWo2bN2+OBQsWxA9/+MMj9iZYnZtvvjneeOONePrpp2PZsmUN1mL+LA7VKw+lvLw8Nm7cGDt27Kjf9uabb0aLFi3cLEGzJbCEZuyEE06IH/zgBzFp0qQ4//zzs5php06donXr1rFs2bLYsmVL/ZfNTJo0KW688cZYvHhxrFmzJiZPnhzV1dVRWVl50HNVVVXFrbfeGrNnz461a9fGa6+9Fr/5zW+O6vUdyl133RV//OMfY+3atbF27dq477774pZbbonhw4fXz6moqIgHHnggFi5cGG3bto3NmzfH5s2bLYAN8D/NvZdERFRXV0d1dXXs2LEj/v3vf0d1dXWsWrWqfnzs2LFRU1MTlZWV8eabb8ajjz4a119/fVRUVDRi1QCN784774za2toYPHhwrFixIjZs2BDLli2L8847L0pKSmLmzJlH9HwrV66Ma6+9Nu69994oLy+PWbNmRWVlZbz11ltH9DyftGrVqqiuro6amprYtm1bfd+oc8kll0THjh1j1KhRsWrVqlixYkVMmjQpRo8e7ePgNFs+Eg7N3E9+8pNYuHBhjB49Omt7Xl5ezJ49O2bMmBHXXnttfOMb34innnoqrrrqqti2bVtcffXVsXXr1ujVq1csXbr0kB9TGDFiROzatStuu+22mDhxYpx44okxdOjQo3lph5TJZGLKlCnx9ttvR15eXnzlK1+JG2+8Ma644or6OXVr1QwcODBr3/nz58fIkSO/wGoB0qs595KIiL59+9b//Oqrr8bChQujtLQ01q9fHxERXbt2jccffzzGjx8fZ555ZpSUlERlZWVcc801jVQxQDp07949XnnllZg+fXoMGzYsampqoqioKIYMGRLTp0+PDh06HLFz7dq1K4YPHx4jR46M7373uxERcfnll8ejjz4aP/7xj2PFihVZS3ccSRdccEG888479c/r+kbd+p3HHXdcPPHEE/Gzn/0s+vXrFx07doxhw4bFr371q6NSDxwLcpJPs8It0OTcf//9MX78+Ni4caOPpQHwmeglAAAcSe6whGbqww8/jE2bNsWvf/3ruOKKK7zABOBT00sAADgarGEJzdRNN90Up556ahQVFcWUKVMauxwAjkF6CQAAR4OPhAMAAAAAqeEOSwAAAAAgNQSWAAAAAEBqCCwBAAAAgNQQWAIAAAAAqSGwBAAAAABSQ2AJAAAAAKSGwBIAAAAASA2BJQAAAACQGgJLAAAAACA1/g8Bo8IKA5OcAwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1600x1200 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Idlu0heP_uH8"
      },
      "source": [
        "**Compare output**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FNF8M5kQR4Mr"
      },
      "source": [
        "diff_fp32 = 0\n",
        "diff_fp16 = 0\n",
        "total = 0\n",
        "\n",
        "for seq in range(pytorch_embedding.shape[0]):\n",
        "  for token in range(pytorch_embedding.shape[1]):\n",
        "    for feature in range(pytorch_embedding.shape[2]):\n",
        "      diff_fp32 += abs(abs(pytorch_embedding[seq][token][feature]) - abs(onnx_embedding[seq][token][feature]))\n",
        "      diff_fp16 += abs(abs(pytorch_embedding_half[seq][token][feature]) - abs(onnx_embedding[seq][token][feature]))\n",
        "      total += 1\n",
        "\n",
        "avg_fp32 = diff_fp32 / total\n",
        "avg_fp16 = diff_fp16 / total"
      ],
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w72FxsIebK5L",
        "outputId": "f23563ac-86da-48af-92fb-2320300cd11a"
      },
      "source": [
        "print(\"Average difference between pytorch fp32 embedding and onnx embedding = {0}\".format(avg_fp32))\n",
        "print(\"Average difference between pytorch fp16 embedding and onnx embedding = {0}\".format(avg_fp16))"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Average difference between pytorch fp32 embedding and onnx embedding = 0.0004189802275504917\n",
            "Average difference between pytorch fp16 embedding and onnx embedding = 6.300210952758789e-05\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}